Repository: ambari Updated Branches: refs/heads/branch-dev-logsearch cb25daa33 -> 69adbd527
Updates to LogSearch Stack service definition, including role_command_order changes (Oliver Szabo via rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/69adbd52 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/69adbd52 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/69adbd52 Branch: refs/heads/branch-dev-logsearch Commit: 69adbd5270582aa51151dd87b69475e92b1d7f6e Parents: cb25daa Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Wed Mar 2 15:29:03 2016 -0500 Committer: Bob Nettleton <rnettle...@hortonworks.com> Committed: Wed Mar 2 15:29:03 2016 -0500 ---------------------------------------------------------------------- .../LOGSEARCH/0.5.0/metainfo.xml | 189 ++++++++++--------- .../LOGSEARCH/0.5.0/metrics.json | 59 ++++++ .../0.5.0/package/scripts/logfeeder.py | 11 +- .../0.5.0/package/scripts/logsearch.py | 11 +- .../LOGSEARCH/0.5.0/package/scripts/solr.py | 11 +- .../0.5.0/package/scripts/status_params.py | 1 - .../LOGSEARCH/0.5.0/widgets.json | 173 +++++++++++++++++ .../stacks/HDP/2.4/role_command_order.json | 3 + 8 files changed, 347 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index 8d7f665..3429304 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -16,98 +16,103 @@ limitations under the License. --> <metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>LOGSEARCH</name> - <displayName>Log Search</displayName> - <comment>Log aggregation, analysis, and visualization for Hadoop</comment> - <version>0.5.0</version> - + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <displayName>Log Search</displayName> + <comment>Log aggregation, analysis, and visualization for Hadoop</comment> + <version>0.5.0</version> - <components> - - <component> - <name>LOGSEARCH_MASTER</name> - <timelineAppid>logsearch</timelineAppid> - <displayName>Log Search</displayName> - <category>MASTER</category> - <cardinality>1</cardinality> - <commandScript> - <script>scripts/logsearch.py</script> - <scriptType>PYTHON</scriptType> - <timeout>5000</timeout> - </commandScript> - <customCommands> - <customCommand> - <name>UPDATE_LOGSEARCH</name> - <commandScript> - <script>scripts/logsearch.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </customCommand> - </customCommands> + <components> - </component> - - <component> - <name>LOGSEARCH_SOLR</name> - <displayName>Solr</displayName> - <category>MASTER</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/solr.py</script> - <scriptType>PYTHON</scriptType> - <timeout>5000</timeout> - </commandScript> - </component> - - <component> - <name>LOGSEARCH_LOGFEEDER</name> - <timelineAppid>logfeeder</timelineAppid> - <displayName>Log Feeder</displayName> - <category>SLAVE</category> - <cardinality>ALL</cardinality> - <commandScript> - <script>scripts/logfeeder.py</script> - <scriptType>PYTHON</scriptType> - <timeout>5000</timeout> - </commandScript> - - <customCommands> - <customCommand> - <name>UPDATE_LOGFEEDER</name> - <commandScript> - <script>scripts/logfeeder.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </customCommand> - </customCommands> - - </component> - - </components> - - <osSpecifics> - <osSpecific> - <osFamily>redhat6</osFamily> - <packages> - <package><name>wget</name></package> - <package><name>tar</name></package> - <package><name>zip</name></package> - </packages> - </osSpecific> - </osSpecifics> - <configuration-dependencies> - <config-type>logsearch-config</config-type> - <config-type>solr-config</config-type> - <config-type>logfeeder-config</config-type> - </configuration-dependencies> - <restartRequiredAfterChange>false</restartRequiredAfterChange> - <widgetsFileName>widgets.json</widgetsFileName> - <metricsFileName>metrics.json</metricsFileName> - </service> - </services> + <component> + <name>LOGSEARCH_MASTER</name> + <timelineAppid>logsearch</timelineAppid> + <displayName>Log Search</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <commandScript> + <script>scripts/logsearch.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + <customCommands> + <customCommand> + <name>UPDATE_LOGSEARCH</name> + <commandScript> + <script>scripts/logsearch.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + + </component> + + <component> + <name>LOGSEARCH_SOLR</name> + <displayName>Solr</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/solr.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + </component> + + <component> + <name>LOGSEARCH_LOGFEEDER</name> + <timelineAppid>logfeeder</timelineAppid> + <displayName>Log Feeder</displayName> + <category>SLAVE</category> + <cardinality>ALL</cardinality> + <commandScript> + <script>scripts/logfeeder.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + + <customCommands> + <customCommand> + <name>UPDATE_LOGFEEDER</name> + <commandScript> + <script>scripts/logfeeder.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + + </component> + + </components> + + <osSpecifics> + <osSpecific> + <osFamily>redhat6</osFamily> + <packages> + <package> + <name>wget</name> + </package> + <package> + <name>tar</name> + </package> + <package> + <name>zip</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + <configuration-dependencies> + <config-type>logsearch-config</config-type> + <config-type>solr-config</config-type> + <config-type>logfeeder-config</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>false</restartRequiredAfterChange> + <widgetsFileName>widgets.json</widgetsFileName> + <metricsFileName>metrics.json</metricsFileName> + </service> + </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json new file mode 100644 index 0000000..a6c20c7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json @@ -0,0 +1,59 @@ +{ + "LOGSEARCH_LOGFEEDER": { + "Component": [ + { + "type": "ganglia", + "metrics": { + "default": { + "metrics/filter/error/grok": { + "metric": "filter.error.grok", + "pointInTime": false, + "temporal": true + }, + "metrics/filter/error/truncate": { + "metric": "filter.error.truncate", + "pointInTime": false, + "temporal": true + }, + "metrics/input/files/count": { + "metric": "input.files.count", + "pointInTime": true, + "temporal": true + }, + "metrics/input/files/read_lines": { + "metric": "input.files.read_lines", + "pointInTime": false, + "temporal": true + }, + "metrics/input/files/read_bytes": { + "metric": "input.files.read_bytes", + "pointInTime": false, + "temporal": true + }, + "metrics/output/kafka/write_logs": { + "metric": "output.kafka.write_logs", + "pointInTime": false, + "temporal": true + }, + "metrics/output/kafka/write_bytes": { + "metric": "output.kafka.write_bytes", + "pointInTime": false, + "temporal": true + }, + "metrics/output/solr/write_logs": { + "metric": "output.solr.write_logs", + "pointInTime": false, + "temporal": true + }, + "metrics/output/solr/write_bytes": { + "metric": "output.solr.write_bytes", + "pointInTime": false, + "temporal": true + } + + } + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py index 8da6b82..052ce52 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py @@ -19,9 +19,8 @@ limitations under the License. import sys, os, pwd, grp, signal, time from resource_management import * -from subprocess import call -class Master(Script): +class LogFeeder(Script): #Call setup.sh to install the service def install(self, env): @@ -72,7 +71,7 @@ class Master(Script): Execute('cd ' + params.logfeeder_dir + '; tar -xvf logfeeder.tar.gz', user=params.logfeeder_user) - def configure(self, env): + def configure(self, env, upgrade_type=None): import params env.set_params(params) @@ -89,7 +88,7 @@ class Master(Script): #Call start.sh to start the service - def start(self, env): + def start(self, env, upgrade_type=None): #import properties defined in -config.xml file from params class import params @@ -114,7 +113,7 @@ class Master(Script): Execute(cmd, user=params.logfeeder_user) #Called to stop the service using the pidfile - def stop(self, env): + def stop(self, env, upgrade_type=None): import params #import status properties defined in -env.xml file from status_params class @@ -152,4 +151,4 @@ class Master(Script): if __name__ == "__main__": - Master().execute() + LogFeeder().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py index 594d85f..3f4f02e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py @@ -19,9 +19,8 @@ limitations under the License. import sys, os, pwd, grp, signal, time from resource_management import * -from subprocess import call -class Master(Script): +class LogSearch(Script): #Call setup.sh to install the service def install(self, env): @@ -72,7 +71,7 @@ class Master(Script): Execute('cd ' + params.logsearch_dir + '; tar -xvf logsearch-portal.tar.gz', user=params.logsearch_user) - def configure(self, env): + def configure(self, env, upgrade_type=None): import params env.set_params(params) @@ -93,7 +92,7 @@ class Master(Script): File(format("{params.logsearch_dir}/solr_configsets/audit_logs/conf/solrconfig.xml"), content=file_content, owner=params.logsearch_user) #Call start.sh to start the service - def start(self, env): + def start(self, env, upgrade_type=None): #import properties defined in -config.xml file from params class import params @@ -151,7 +150,7 @@ class Master(Script): Execute(cmd, user=params.logsearch_user) #Called to stop the service using the pidfile - def stop(self, env): + def stop(self, env, upgrade_type=None): import params #import status properties defined in -env.xml file from status_params class @@ -188,4 +187,4 @@ class Master(Script): self.start(env) if __name__ == "__main__": - Master().execute() + LogSearch().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py index cdbfabd..ec8eeb2 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py @@ -19,9 +19,8 @@ limitations under the License. import sys, os, pwd, grp, signal, time from resource_management import * -from subprocess import call -class Master(Script): +class Solr(Script): #Call setup_solr.sh to install the service def install(self, env): @@ -88,7 +87,7 @@ class Master(Script): - def configure(self, env): + def configure(self, env, upgrade_type=None): import params env.set_params(params) @@ -111,7 +110,7 @@ class Master(Script): #Call start.sh to start the service - def start(self, env): + def start(self, env, upgrade_type=None): #import properties defined in -config.xml file from params class import params @@ -143,7 +142,7 @@ class Master(Script): Execute(cmd, user=params.solr_user) #Called to stop the service using the pidfile - def stop(self, env): + def stop(self, env, upgrade_type=None): import params #import status properties defined in -env.xml file from status_params class @@ -174,4 +173,4 @@ class Master(Script): if __name__ == "__main__": - Master().execute() + Solr().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py index f9ed87c..86e8f2e 100755 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py @@ -20,7 +20,6 @@ limitations under the License. """ from resource_management import * -import glob # config object that holds the status related configurations declared in the -env.xml file config = Script.get_config() http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json new file mode 100644 index 0000000..379122a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json @@ -0,0 +1,173 @@ +{ + "layouts": [ + { + "layout_name": "default_logsearch_dashboard", + "display_name": "Logsearch Dashboard", + "section_name": "LOGSEARCH_SUMMARY", + "widgetLayoutInfo": [ + { + "widget_name": "Files Monitoring", + "description": "Files that are been monitored", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "input.files.count", + "metric_path": "metrics/input/files/count", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + } + ], + "values": [ + { + "name": "Files count", + "value": "${input.files.count}" + } + ], + "properties": { + } + }, + { + "widget_name": "Logs Published", + "description": "Logs Published", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "output.solr.write_logs._sum", + "metric_path": "metrics/output/solr/write_logs", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + }, + { + "name": "output.kafka.write_logs._sum", + "metric_path": "metrics/output/kafka/write_logs", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + } + ], + "values": [ + { + "name": "Logs to Solr", + "value": "${output.solr.write_logs._sum/1000}" + }, + { + "name": "Logs to Kafka", + "value": "${output.kafka.write_logs._sum/1000}" + } + ], + "properties": { + "display_unit": "K", + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Lines Read", + "description": "Lines Read", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "input.files.read_lines._sum", + "metric_path": "metrics/input/files/read_lines", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + } + + ], + "values": [ + { + "name": "Lines read from File", + "value": "${input.files.read_lines._sum/1024}" + } + ], + "properties": { + "display_unit": "K", + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Read/Write Bytes", + "description": "Bytes read and written", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "input.files.read_bytes._sum", + "metric_path": "metrics/input/files/read_bytes", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + }, + { + "name": "output.solr.write_bytes._sum", + "metric_path": "metrics/output/solr/write_bytes", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + }, + { + "name": "output.kafka.write_bytes._sum", + "metric_path": "metrics/output/kafka/write_bytes", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + } + ], + "values": [ + { + "name": "Data read from File", + "value": "${input.files.read_bytes._sum/(1024*1024)}" + }, + { + "name": "Data sent to Solr", + "value": "${output.solr.write_bytes._sum/(1024*1024)}" + }, + { + "name": "Data sent to Kafka", + "value": "${output.kafka.write_bytes._sum/(1024*1024)}" + } + ], + "properties": { + "display_unit": "MB", + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Parse Errors", + "description": "Parse Errors", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "filter.error.grok._sum", + "metric_path": "metrics/filter/error/grok", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + }, + { + "name": "filter.error.truncate._sum", + "metric_path": "metrics/filter/error/truncate", + "service_name": "LOGSEARCH", + "component_name": "LOGSEARCH_LOGFEEDER" + } + ], + "values": [ + { + "name": "Grok Errors", + "value": "${filter.error.grok._sum}" + }, + { + "name": "Logs Truncated", + "value": "${filter.error.truncate._sum}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json index 14b2563..9e13808 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json @@ -3,6 +3,9 @@ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", "general_deps" : { "_comment" : "dependencies for all cases", + "LOGSEARCH_SOLR-START" : ["ZOOKEEPER_SERVER-START"], + "LOGSEARCH_MASTER-START": ["LOGSEARCH_SOLR-START"], + "LOGSEARCH_LOGFEEDER-START": ["LOGSEARCH_SOLR-START", "LOGSEARCH_MASTER-START"], "SPARK_THRIFTSERVER-START" : ["HIVE_SERVER-START"] } }
