http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 index efa51fa..293a772 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 @@ -15,84 +15,491 @@ # See the License for the specific language governing permissions and # limitations under the License. #} - { - "service": { -{% if logsearch_metadata and logsearch_metadata.keys()|length > 0 %} -{% for metadata_key, component_list in logsearch_metadata.iteritems() %} - "{{ metadata_key.lower() }}": { - "label": "{{ metadata_key }}", +{ + "service": { + "accumulo": { + "label": "Accumulo", "components": [ -{% for component in component_list %} { - "name": "{{ component }}" - }{% if not loop.last %},{% endif %} - -{% endfor %} + "name": "accumulo_gc" + }, + { + "name": "accumulo_master" + }, + { + "name": "accumulo_monitor" + }, + { + "name": "accumulo_tracer" + }, + { + "name": "accumulo_tserver" + } ], "dependencies": [ ] - } - , -{% endfor %} -{% endif %} -{% if logfeeder_system_log_enabled %} - "system" : { - "label" : "System", - "components" : [ + }, + "atlas": { + "label": "Atlas", + "components": [ + { + "name": "atlas_app" + } + ], + "dependencies": [ + ] + }, + "ambari": { + "label": "Ambari", + "components": [ + { + "name": "ambari_agent" + }, + { + "name": "ambari_server" + }, + { + "name": "ambari_alerts" + }, + { + "name": "ambari_audit" + }, + { + "name": "ambari_config_changes" + }, + { + "name": "ambari_eclipselink" + }, + { + "name": "ambari_server_check_database" + } + ], + "dependencies": [ + ] + }, + "ams": { + "label": "AMS", + "components": [ + { + "name": "ams_hbase_master" + }, + { + "name": "ams_hbase_regionserver" + }, + { + "name": "ams_collector" + }, + { + "name": "ams_monitor" + }, + { + "name": "ams_grafana" + } + ], + "dependencies": [ + ] + }, + "falcon": { + "label": "Falcon", + "components": [ + { + "name": "falcon_app" + } + ], + "dependencies": [ + ] + }, + "flume": { + "label": "Flume", + "components": [ + { + "name": "flume_handler" + } + ], + "dependencies": [ + ] + }, + "hbase": { + "label": "HBase", + "components": [ + { + "name": "hbase_master" + }, + { + "name": "hbase_regionserver" + }, + { + "name": "hbase_phoenix_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, + "hdfs": { + "label": "HDFS", + "components": [ + { + "name": "hdfs_datanode" + }, + { + "name": "hdfs_namenode" + }, + { + "name": "hdfs_journalnode" + }, + { + "name": "hdfs_secondarynamenode" + }, + { + "name": "hdfs_zkfc" + }, + { + "name": "hdfs_nfs3" + }, + { + "name": "hdfs_audit", + "rowtype": "audit" + } + ], + "dependencies": [ + ] + }, + "hive": { + "label": "Hive", + "components": [ + { + "name": "hive_hiveserver2" + }, + { + "name": "hive_metastore" + }, + { + "name": "webhcat_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, + "infra" : { + "label" : "Infra", + "components": [ + { + "name": "infra_solr" + } + ], + "dependencies": [ + ] + }, + "kafka": { + "label": "Kafka", + "components": [ + { + "name": "kafka_controller" + }, + { + "name": "kafka_request" + }, + { + "name": "kafka_logcleaner" + }, + { + "name": "kafka_server" + }, + { + "name": "kafka_statechange" + } + ], + "dependencies": [ + { + "service": "zookeeper", + "components": [ + "zookeeper" + ] + } + ] + }, + "knox": { + "label": "Knox", + "components": [ + { + "name": "knox_gateway" + }, + { + "name": "knox_cli" + }, + { + "name": "knox_ldap" + } + ], + "dependencies": [ + ] + }, + "nifi": { + "label": "NiFi", + "components": [ + { + "name": "nifi_app" + }, + { + "name": "nifi_bootstrap" + }, + { + "name": "nifi_setup" + }, + { + "name": "nifi_user" + } + ], + "dependencies": [ + ] + }, + "mapred": { + "label": "MapReduce", + "components": [ + { + "name": "mapred_historyserver" + } + ], + "dependencies": [ + ] + }, + "logsearch": { + "label": "Logsearch", + "components": [ + { + "name": "logsearch_app" + }, + { + "name": "logsearch_feeder" + }, + { + "name": "logsearch_perf" + }, + { + "name": "logsearch_solr" + } + ], + "dependencies": [ + ] + }, + "ranger": { + "label": "Ranger", + "components": [ + { + "name": "ranger_admin" + }, + { + "name": "ranger_dbpatch" + }, + { + "name": "ranger_kms" + }, + { + "name": "ranger_usersync" + } + ], + "dependencies": [ + { + "service": "hdfs", + "required": "optional", + "components": [ + "hdfs_namenode" + ] + }, + { + "service": "hbase", + "required": "optional", + "components": [ + "hbase_master", + "hbase_regionserver" + ] + }, + { + "service": "hive", + "required": "optional", + "components": [ + "hive_hiveserver2" + ] + }, + { + "service": "kafka", + "required": "optional", + "components": [ + "kafka_ranger" + ] + }, + { + "service": "knox", + "required": "optional", + "components": [ + "knox_gateway" + ] + }, + { + "service": "storm", + "required": "optional", + "components": [ + "storm_supervisor" + ] + }, + { + "service": "yarn", + "required": "optional", + "components": [ + "yarn_resourcemanager" + ] + } + ] + }, + "oozie": { + "label": "Oozie", + "components": [ + { + "name": "oozie_app" + } + ], + "dependencies": [ + ] + }, + "hst": { + "label": "SmartSense", + "components": [ + { + "name": "hst_server" + }, + { + "name": "hst_agent" + }, + { + "name": "activity_analyzer" + }, + { + "name": "activity_explorer" + } + ], + "dependencies": [ + ] + }, + "spark": { + "label": "Spark", + "components": [ + { + "name": "spark_jobhistory_server" + }, + { + "name": "spark_thriftserver" + }, + { + "name": "livy_server" + } + ], + "dependencies": [ + ] + }, + "spark2": { + "label": "Spark 2", + "components": [ + { + "name": "spark2_jobhistory_server" + }, + { + "name": "spark2_thriftserver" + } + ], + "dependencies": [ + ] + }, + "storm": { + "label": "Storm", + "components": [ + { + "name": "storm_drpc" + }, + { + "name": "storm_logviewer" + }, + { + "name": "storm_nimbus" + }, + { + "name": "storm_supervisor" + }, + { + "name": "storm_ui" + }, + { + "name": "storm_worker" + } + ], + "dependencies": [ + ] + }, + "yarn": { + "label": "YARN", + "components": [ + { + "name": "yarn_nodemanager" + }, + { + "name": "yarn_resourcemanager" + }, + { + "name": "yarn_timelineserver" + }, + { + "name": "yarn_historyserver" + }, + { + "name": "yarn_jobsummary" + } + ], + "dependencies": [ + ] + }, + "zeppelin": { + "label": "Zeppelin", + "components": [ + { + "name": "zeppelin" + } + ], + "dependencies": [ + ] + }, + "zookeeper": { + "label": "ZooKeeper", + "components": [ + { + "name": "zookeeper" + } + ], + "dependencies": [ + ] + }, + "System": { + "label": "System", + "components": [ { "name": "system_message" }, { "name": "secure_log" } - ], - "dependencies": [ - ] - }, -{% endif %} - "ambari": { - "label": "Ambari", - "components": [ - { - "name": "ambari_agent" - }, - { - "name": "ambari_server" - }, - { - "name": "ambari_alerts" - }, - { - "name": "ambari_audit" - }, - { - "name": "ambari_config_changes" - }, - { - "name": "ambari_eclipselink" - }, - { - "name": "ambari_server_check_database" - } - ], - "dependencies": [ - ] - }, - "logsearch": { - "label": "Logsearch", - "components": [ - { - "name": "logsearch_app" - }, - { - "name": "logsearch_feeder" - }, - { - "name": "logsearch_perf" - } - ], - "dependencies": [ - ] - } + ], + "dependencies": [ + ] + } } -} \ No newline at end of file +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 index ea91405..25bfa63 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 @@ -20,17 +20,17 @@ { "type":"logsearch_app", "rowtype":"service", - "path":"{{logsearch_log_dir}}/logsearch.json" + "path":"{{default('/configurations/logsearch-env/logsearch_log_dir', '/var/log/ambari-logsearch-portal')}}/logsearch.json" }, { "type":"logsearch_feeder", "rowtype":"service", - "path":"{{logfeeder_log_dir}}/logsearch-logfeeder.json" + "path":"{{default('/configurations/logfeeder-env/logfeeder_log_dir', '/var/log/ambari-logsearch-logfeeder')}}/logsearch-logfeeder.json" }, { "type":"logsearch_perf", "rowtype":"service", - "path":"{{logsearch_log_dir}}/logsearch-performance.json" + "path":"{{default('/configurations/logfeeder-env/logfeeder_log_dir', '/var/log/ambari-logsearch-logfeeder')}}/logsearch-performance.json" } ], "filter":[ http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml deleted file mode 100644 index 2f13d3f..0000000 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml +++ /dev/null @@ -1,80 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Oozie</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>OOZIE_SERVER:oozie_app</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"oozie_app", - "rowtype":"service", - "path":"{{default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')}}/oozie.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "oozie_app" - ] - } - }, - "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/input.config-oozie.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/input.config-oozie.json.j2 new file mode 100644 index 0000000..4a54f74 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/input.config-oozie.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')}}/oozie.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + } + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml deleted file mode 100644 index fc26336..0000000 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml +++ /dev/null @@ -1,111 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Ranger</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>RANGER_SERVER:ranger_admin,ranger_dbpatch;RANGER_USERSYNC:ranger_usersync;</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"ranger_admin", - "rowtype":"service", - "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log" - }, - { - "type":"ranger_dbpatch", - "is_enabled":"true", - "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" - }, - { - "type":"ranger_usersync", - "rowtype":"service", - "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_admin", - "ranger_dbpatch" - ] - } - }, - "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_usersync" - ] - } - }, - "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", - "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", - "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 new file mode 100644 index 0000000..6c5bb1f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 @@ -0,0 +1,79 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_admin", + "ranger_dbpatch" + ] + } + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + } + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml deleted file mode 100644 index 226d2b0..0000000 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml +++ /dev/null @@ -1,80 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Ranger KMS</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>RANGER_KMS:ranger_kms</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"ranger_kms", - "rowtype":"service", - "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_kms" - ] - } - }, - "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/templates/input.config-ranger-kms.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/templates/input.config-ranger-kms.json.j2 b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/templates/input.config-ranger-kms.json.j2 new file mode 100644 index 0000000..306fade --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/templates/input.config-ranger-kms.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml deleted file mode 100644 index 63201ef..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml +++ /dev/null @@ -1,98 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Spark</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>SPARK_JOBHISTORYSERVER:spark_jobhistory_server;SPARK_THRIFTSERVER:spark_thriftserver;LIVY_SERVER:livy_server</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"spark_jobhistory_server", - "rowtype":"service", - "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" - }, - { - "type":"spark_thriftserver", - "rowtype":"service", - "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" - }, - { - "type":"livy_server", - "rowtype":"service", - "path":"{{default('/configurations/livy-env/livy_log_dir', '/var/log/livy')}}/livy-livy-server.out" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "spark_jobhistory_server", - "spark_thriftserver", - "livy_server" - ] - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", - "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yy/MM/dd HH:mm:ss" - } - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - } - } - } - ] -} - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/templates/input.config-spark.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/templates/input.config-spark.json.j2 new file mode 100644 index 0000000..7eea751 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/templates/input.config-spark.json.j2 @@ -0,0 +1,66 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"spark_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + }, + { + "type":"livy_server", + "rowtype":"service", + "path":"{{default('/configurations/livy-env/livy_log_dir', '/var/log/livy')}}/livy-livy-server.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark_jobhistory_server", + "spark_thriftserver", + "livy_server" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml deleted file mode 100644 index cb71c6b..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml +++ /dev/null @@ -1,98 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Spark2</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>SPARK2_JOBHISTORYSERVER:spark2_jobhistory_server;SPARK2_THRIFTSERVER:spark2_thriftserver;LIVY2_SERVER:livy2_server</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"spark2_jobhistory_server", - "rowtype":"service", - "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" - }, - { - "type":"spark2_thriftserver", - "rowtype":"service", - "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" - }, - { - "type":"livy2_server", - "rowtype":"service", - "path":"{{default('/configurations/livy2-env/livy2_log_dir', '/var/log/livy2')}}/livy-livy-server.out" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "spark2_jobhistory_server", - "spark2_thriftserver", - "livy2_server" - ] - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", - "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yy/MM/dd HH:mm:ss" - } - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - } - } - } - ] -} - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/templates/input.config-spark2.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/templates/input.config-spark2.json.j2 b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/templates/input.config-spark2.json.j2 new file mode 100644 index 0000000..e90aa65 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/templates/input.config-spark2.json.j2 @@ -0,0 +1,66 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"spark2_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark2_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + }, + { + "type":"livy2_server", + "rowtype":"service", + "path":"{{default('/configurations/livy2-env/livy2_log_dir', '/var/log/livy2')}}/livy-livy-server.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark2_jobhistory_server", + "spark2_thriftserver", + "livy2_server" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml deleted file mode 100644 index d485017..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml +++ /dev/null @@ -1,110 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Storm</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>NIMBUS:storm_nimbus;SUPERVISOR:storm_supervisor,storm_worker,storm_logviewer;STORM_UI_SERVER:storm_ui;DRPC_SERVER:storm_drpc</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"storm_drpc", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/drpc.log" - }, - { - "type":"storm_logviewer", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/logviewer.log" - }, - { - "type":"storm_nimbus", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/nimbus.log" - }, - { - "type":"storm_supervisor", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/supervisor.log" - }, - { - "type":"storm_ui", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/ui.log" - }, - { - "type":"storm_worker", - "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "storm_drpc", - "storm_logviewer", - "storm_nimbus", - "storm_supervisor", - "storm_ui", - "storm_worker" - ] - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 new file mode 100644 index 0000000..a2a4841 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 @@ -0,0 +1,78 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml deleted file mode 100644 index 3c0abbf..0000000 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml +++ /dev/null @@ -1,80 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>MapReduce</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>HISTORYSERVER:mapred_historyserver</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"mapred_historyserver", - "rowtype":"service", - "path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "mapred_historyserver" - ] - } - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml deleted file mode 100644 index 95cf0c9..0000000 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml +++ /dev/null @@ -1,104 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>YARN</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>RESOURCEMANAGER:yarn_resourcemanager,yarn_historyserver,yarn_jobsummary;NODEMANAGER:yarn_nodemanager;APP_TIMELINE_SERVER:yarn_timelineserver</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"yarn_nodemanager", - "rowtype":"service", - "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log" - }, - { - "type":"yarn_resourcemanager", - "rowtype":"service", - "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log" - }, - { - "type":"yarn_timelineserver", - "rowtype":"service", - "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log" - }, - { - "type":"yarn_historyserver", - "rowtype":"service", - "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log" - }, - { - "type":"yarn_jobsummary", - "rowtype":"service", - "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "yarn_historyserver", - "yarn_jobsummary", - "yarn_nodemanager", - "yarn_resourcemanager", - "yarn_timelineserver" - ] - } - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] -} - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 new file mode 100644 index 0000000..8034843 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "mapred_historyserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-yarn.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-yarn.json.j2 new file mode 100644 index 0000000..bf1dd37 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-yarn.json.j2 @@ -0,0 +1,72 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml deleted file mode 100644 index 0b8ab7a..0000000 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml +++ /dev/null @@ -1,80 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Zeppelin</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>ZEPPELIN_MASTER:zeppelin</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"zeppelin", - "rowtype":"service", - "path":"{{default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')}}/zeppelin-zeppelin-*.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "zeppelin" - ] - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", - "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] - } - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/templates/input.config-zeppelin.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/templates/input.config-zeppelin.json.j2 b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/templates/input.config-zeppelin.json.j2 new file mode 100644 index 0000000..2b373d5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/templates/input.config-zeppelin.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"zeppelin", + "rowtype":"service", + "path":"{{default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')}}/zeppelin-zeppelin-*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zeppelin" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml deleted file mode 100644 index 325af14..0000000 --- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml +++ /dev/null @@ -1,76 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="false" supports_adding_forbidden="true"> - <property> - <name>service_name</name> - <display-name>Service name</display-name> - <description>Service name for Logsearch Portal (label)</description> - <value>Zookeeper</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>component_mappings</name> - <display-name>Component mapping</display-name> - <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> - <value>ZOOKEEPER_SERVER:zookeeper</value> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>content</name> - <display-name>Logfeeder Config</display-name> - <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> - <value> -{ - "input":[ - { - "type":"zookeeper", - "rowtype":"service", - "path":"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{"type":["zookeeper"]} - }, - "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values": { - "logtime": { - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - } - } - ] -} - </value> - <value-attributes> - <type>content</type> - <show-property-name>false</show-property-name> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/templates/input.config-zookeeper.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/templates/input.config-zookeeper.json.j2 new file mode 100644 index 0000000..cc20847 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/templates/input.config-zookeeper.json.j2 @@ -0,0 +1,46 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":["zookeeper"] + } + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py index 1782298..fbea258 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py @@ -27,6 +27,7 @@ from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_stack_version +from string import lower config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -44,6 +45,19 @@ stack_version_formatted = format_stack_version(stack_version_unformatted) # current host stack version current_version = default("/hostLevelParams/current_version", None) +# service name +service_name = config['serviceName'] + +# logsearch configuration +logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf" + +agent_cache_dir = config['hostLevelParams']['agentCacheDir'] +service_package_folder = config['commandParams']['service_package_folder'] +logsearch_service_name = service_name.lower().replace("_", "-") +logsearch_config_file_name = 'input.config-' + logsearch_service_name + ".json" +logsearch_config_file_path = agent_cache_dir + "/" + service_package_folder + "/templates/" + logsearch_config_file_name + ".j2" +logsearch_config_file_exists = os.path.exists(logsearch_config_file_path) + # default hadoop params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py index e9f2283..36a202f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py @@ -19,9 +19,13 @@ limitations under the License. import os import ambari_simplejson as json +from ambari_jinja2 import Environment as JinjaEnvironment from resource_management.core.logger import Logger +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import InlineTemplate, Template from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock @@ -72,6 +76,19 @@ def setup_config(): group=params.user_group, only_if=format("ls {hadoop_conf_dir}")) + Directory(params.logsearch_logfeeder_conf, + mode=0755, + cd_access='a', + create_parents=True + ) + + if params.logsearch_config_file_exists: + File(format("{logsearch_logfeeder_conf}/" + params.logsearch_config_file_name), + content=Template(params.logsearch_config_file_path,extra_imports=[default]) + ) + else: + Logger.warning('No logsearch configuration exists at ' + params.logsearch_config_file_path) + def load_version(struct_out_file): """