Repository: ambari Updated Branches: refs/heads/branch-metrics-dev 3c23bee67 -> a3ed7a3fd
http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py index c31caf2..6488a79 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py @@ -26,6 +26,23 @@ import status_params config = Script.get_config() exec_tmp_dir = Script.get_tmp_dir() +#AMS data +ams_user=status_params.ams_user +ams_pid_dir="/var/run/ambari-metrics" + +ams_collector_script = "/usr/sbin/ambari-metrics-collector" +ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf" +ams_collector_pid_dir = status_params.ams_collector_pid_dir +ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", []) +ams_collector_host_single = ams_collector_hosts[0] #TODO cardinality is 1+ so we can have more than one host +ams_collector_port = 8188 + + +ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/" +ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring" +ams_monitor_pid_dir = status_params.ams_monitor_pid_dir +ams_monitor_script = "/usr/sbin/ambari-metrics-monitor" + #RPM versioning support rpm_version = default("/configurations/hadoop-env/rpm_version", None) @@ -36,12 +53,14 @@ if rpm_version is not None: #hadoop params if rpm_version is not None: + hadoop_native_lib = format("/usr/hdp/current/hadoop-client/lib/native/") hadoop_bin_dir = format("/usr/hdp/current/hadoop/bin") daemon_script = format('/usr/hdp/current/hbase/bin/hbase-daemon.sh') region_mover = format('/usr/hdp/current/hbase/bin/region_mover.rb') - region_drainer = format('/usr/hdp/currenthbase/bin/draining_servers.rb') + region_drainer = format('/usr/hdp/current/hbase/bin/draining_servers.rb') hbase_cmd = format('/usr/hdp/current/hbase/bin/hbase') else: + hadoop_native_lib = format("/usr/lib/hadoop/lib/native") hadoop_bin_dir = "/usr/bin" daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh" region_mover = "/usr/lib/hbase/bin/region_mover.rb" @@ -49,7 +68,8 @@ else: hbase_cmd = "/usr/lib/hbase/bin/hbase" hadoop_conf_dir = "/etc/hadoop/conf" -hbase_conf_dir = "/etc/hbase/conf" +#hbase_conf_dir = "/etc/ams-hbase/conf" +hbase_conf_dir = ams_collector_conf_dir hbase_excluded_hosts = config['commandParams']['excluded_hosts'] hbase_drain_only = config['commandParams']['mark_draining_only'] hbase_included_hosts = config['commandParams']['included_hosts'] @@ -65,32 +85,34 @@ metric_prop_file_name = "hadoop-metrics2-hbase.properties" # not supporting 32 bit jdk. java64_home = config['hostLevelParams']['java_home'] -log_dir = config['configurations']['hbase-env']['hbase_log_dir'] -master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize'] +hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir'] +master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize'] -regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize'] -regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] -regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] +regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize'] +regionserver_xmn_max = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_max'] +regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) -pid_dir = status_params.pid_dir -tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +hbase_pid_dir = status_params.hbase_pid_dir +hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir'] # TODO UPGRADE default, update site during upgrade -_local_dir_conf = default('/configurations/hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local") -local_dir = substitute_vars(_local_dir_conf, config['configurations']['hbase-site']) +_local_dir_conf = default('/configurations/ams-hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local") +local_dir = substitute_vars(_local_dir_conf, config['configurations']['ams-hbase-site']) client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") -ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present -ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0] +# ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present +# ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0] # if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case -if 'slave_hosts' in config['clusterHostInfo']: - rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves -else: - rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') +# if 'slave_hosts' in config['clusterHostInfo']: +# rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves +# else: +# rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') + +rs_hosts = ["localhost"] smoke_test_user = config['configurations']['cluster-env']['smokeuser'] smokeuser_permissions = "RWXCA" @@ -99,28 +121,30 @@ user_group = config['configurations']['cluster-env']["user_group"] if security_enabled: _hostname_lowercase = config['hostname'].lower() - master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) - regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) + master_jaas_princ = config['configurations']['ams-hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) + regionserver_jaas_princ = config['configurations']['ams-hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) -master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] -regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] +master_keytab_path = config['configurations']['ams-hbase-site']['hbase.master.keytab.file'] +regionserver_keytab_path = config['configurations']['ams-hbase-site']['hbase.regionserver.keytab.file'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] +hbase_user_keytab = config['configurations']['ams-hbase-env']['hbase_user_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -if security_enabled: - kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") -else: - kinit_cmd = "" + +# if security_enabled: +# kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") +# else: +# kinit_cmd = "" #log4j.properties -if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): - log4j_props = config['configurations']['hbase-log4j']['content'] +if (('ams-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['ams-hbase-log4j'])): + log4j_props = config['configurations']['ams-hbase-log4j']['content'] else: log4j_props = None - -hbase_env_sh_template = config['configurations']['hbase-env']['content'] -hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] +hbase_env_sh_template = config['configurations']['ams-hbase-env']['content'] +ams_env_sh_template = config['configurations']['ams-env']['content'] + +hbase_hdfs_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir'] hbase_staging_dir = "/apps/hbase/staging" #for create_hdfs_directory hostname = config["hostname"] @@ -131,12 +155,15 @@ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/ import functools #create partial functions with common arguments for every HdfsDirectory call #to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - bin_dir = hadoop_bin_dir -) +# HdfsDirectory = functools.partial( +# HdfsDirectory, +# conf_dir=hadoop_conf_dir, +# hdfs_user=hdfs_user, +# security_enabled = security_enabled, +# keytab = hdfs_user_keytab, +# kinit_path_local = kinit_path_local, +# bin_dir = hadoop_bin_dir +# ) + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py new file mode 100644 index 0000000..0e949eb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir'] +hbase_user = config['configurations']['ams-hbase-env']['hbase_user'] +ams_user = config['configurations']['ams-env']['ams_user'] + +ams_monitor_pid_dir = "/var/run/ambari-metrics-monitor" +ams_collector_pid_dir = "/var/run/ambari-metrics-collector" http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 new file mode 100644 index 0000000..f86343f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 @@ -0,0 +1,55 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{ + "host_metric_groups": { + "cpu_info": { + "collect_every": "15", + "metrics": [ + { + "name": "cpu_user", + "value_threshold": "1.0" + } + ] + }, + "disk_info": { + "collect_every": "30", + "metrics": [ + { + "name": "disk_free", + "value_threshold": "5.0" + } + ] + }, + "network_info": { + "collect_every": "20", + "metrics": [ + { + "name": "bytes_out", + "value_threshold": "128" + } + ] + } + }, + "process_metric_groups": { + "": { + "collect_every": "15", + "metrics": [] + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2 new file mode 100644 index 0000000..5e672fc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2 @@ -0,0 +1,30 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +[default] +debug_level = INFO +metrics_server = {{ams_collector_host_single}}:{{ams_collector_port}} +enable_time_threshold = false +enable_value_threshold = false + +[emitter] +send_interval = 60 + +[collector] +collector_sleep_interval = 5 +max_queue_size = 5000
