Repository: ambari Updated Branches: refs/heads/branch-2.2 5b5ca9353 -> d8920559e
AMBARI-14892 Add Grafana as a component to AMS (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8920559 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8920559 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8920559 Branch: refs/heads/branch-2.2 Commit: d8920559e459c4457584540ab1308879df35d3a9 Parents: 5b5ca93 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Thu Feb 4 12:00:29 2016 -0800 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Fri Feb 5 08:04:14 2016 -0800 ---------------------------------------------------------------------- .../conf/unix/ambari-metrics-grafana | 79 +++--- .../conf/unix/ams-grafana-env.sh | 9 - .../conf/unix/ams-grafana.ini | 10 +- .../timeline/AbstractMiniHBaseClusterTest.java | 1 + .../0.1.0/configuration/ams-grafana-env.xml | 59 ++++ .../0.1.0/configuration/ams-grafana-ini.xml | 282 +++++++++++++++++++ .../AMBARI_METRICS/0.1.0/metainfo.xml | 18 ++ .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 30 ++ .../0.1.0/package/scripts/metrics_collector.py | 2 +- .../0.1.0/package/scripts/metrics_grafana.py | 65 +++++ .../0.1.0/package/scripts/metrics_monitor.py | 2 +- .../0.1.0/package/scripts/params.py | 9 + .../0.1.0/package/scripts/status.py | 4 + .../0.1.0/package/scripts/status_params.py | 1 + .../stacks/HDP/2.0.6/role_command_order.json | 4 +- .../AMBARI_METRICS/test_metrics_grafana.py | 75 +++++ .../python/stacks/2.0.6/configs/default.json | 9 + 17 files changed, 607 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana index 1de5401..29851d2 100644 --- a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana +++ b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana @@ -18,7 +18,7 @@ # description: Grafana web server & backend # processname: grafana # config: /etc/grafana/ams-grafana.ini -# pidfile: /var/run/grafana.pid +# pidfile: /var/run/ambari-metrics-grafana.pid ### BEGIN INIT INFO # Provides: grafana @@ -35,16 +35,18 @@ PATH=/bin:/usr/bin:/sbin:/usr/sbin NAME=grafana-server -DESC="Grafana Server" - -if [ -f "/etc/ambari-metrics-grafana/conf/ams-grafana-env.sh" ]; then - . "/etc/ambari-metrics-grafana/conf/ams-grafana-env.sh" +DESC="Ambari Metrics Grafana" +CONF_DIR=/etc/ambari-metrics-grafana/conf + +# execute ams-grafana-env.sh +if [[ -f "${CONF_DIR}/ams-grafana-env.sh" ]]; then +. "${CONF_DIR}/ams-grafana-env.sh" +else +echo "ERROR: Cannot execute ${CONF_DIR}/ams-grafana-env.sh." 2>&1 +exit 1 fi -GRAFANA_USER=${AMS_USER} -GRAFANA_GROUP=${AMS_USER_GROUP} GRAFANA_HOME=${AMS_GRAFANA_HOME_DIR} -CONF_DIR=${AMS_GRAFANA_CONF_DIR} WORK_DIR=$GRAFANA_HOME DATA_DIR=${AMS_GRAFANA_DATA_DIR} LOG_DIR=${AMS_GRAFANA_LOG_DIR} @@ -52,11 +54,13 @@ CONF_FILE=$CONF_DIR/ams-grafana.ini MAX_OPEN_FILES=10000 PID_FILE=${AMS_GRAFANA_PID_DIR}/$NAME.pid DAEMON=$GRAFANA_HOME/bin/$NAME +OUT_FILE=${AMS_GRAFANA_LOG_DIR}/grafana.out +STOP_TIMEOUT=5 -if [ `id -u` -ne 0 ]; then - echo "You need root privileges to run this script" - exit 4 -fi +#if [ `id -u` -ne 0 ]; then +# echo "You need root privileges to run this script" +# exit 4 +#fi if [ ! -x $DAEMON ]; then echo "Program not installed or not executable" @@ -66,20 +70,20 @@ fi # # init.d / servicectl compatibility (openSUSE) # -if [ -f /etc/rc.status ]; then - . /etc/rc.status - rc_reset -fi +# if [ -f /etc/rc.status ]; then +# . /etc/rc.status +# rc_reset +# fi # # Source function library. # -if [ -f /etc/rc.d/init.d/functions ]; then - . /etc/rc.d/init.d/functions -fi +# if [ -f /etc/rc.d/init.d/functions ]; then +# . /etc/rc.d/init.d/functions +# fi # overwrite settings from default file -[ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME +# [ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR}" @@ -98,16 +102,16 @@ case "$1" in fi # Prepare environment - mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR" - touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE" + # mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR" + # touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE" - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi + # if [ -n "$MAX_OPEN_FILES" ]; then + # ulimit -n $MAX_OPEN_FILES + # fi # Start Daemon cd $GRAFANA_HOME - su -s /bin/sh -c "nohup ${DAEMON} ${DAEMON_OPTS} >> /dev/null 3>&1 &" $GRAFANA_USER 2> /dev/null + nohup ${DAEMON} ${DAEMON_OPTS} > $OUT_FILE 2>&1 & return=$? if [ $return -eq 0 ] then @@ -138,17 +142,22 @@ case "$1" in echo -n "Stopping $DESC ..." if [ -f "$PID_FILE" ]; then - killproc -p $PID_FILE -d 20 $NAME - if [ $? -eq 1 ]; then - echo -n "$DESC is not running but pid file exists, cleaning up" - elif [ $? -eq 3 ]; then - PID="`cat $PID_FILE`" - echo -n "Failed to stop $DESC (pid $PID)" - exit 1 + pid=$(cat "$PID_FILE") + + kill "${pid}" >/dev/null 2>&1 + sleep "${STOP_TIMEOUT}" + + if kill -0 "${pid}" > /dev/null 2>&1; then + echo "WARNING: $DESC did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9" + kill -9 "${pid}" >/dev/null 2>&1 + fi + + if ps -p "${pid}" > /dev/null 2>&1; then + echo "ERROR: Unable to kill ${pid}" + else + rm -f "${pidfile}" >/dev/null 2>&1 fi - rm -f "$PID_FILE" echo "OK" - exit 0 else echo -n "(not running)" fi http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana-env.sh ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana-env.sh b/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana-env.sh index 20986cb..601e930 100644 --- a/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana-env.sh +++ b/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana-env.sh @@ -16,18 +16,9 @@ # Set environment variables here. -# AMS user -export AMS_USER=ams - -# AMS user group -export AMS_USER_GROUP=hadoop - # AMS Grafana Home Dir export AMS_GRAFANA_HOME_DIR=/usr/lib/ambari-metrics-grafana -# AMS Grafana Conf Dir -export AMS_GRAFANA_CONF_DIR=/etc/ambari-metrics-grafana/conf - # AMS Grafana Data Dir export AMS_GRAFANA_DATA_DIR=/var/lib/ambari-metrics-grafana http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana.ini ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana.ini b/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana.ini index e42a92f..c0ccf1a 100644 --- a/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana.ini +++ b/ambari-metrics/ambari-metrics-grafana/conf/unix/ams-grafana.ini @@ -51,7 +51,7 @@ static_root_path = /usr/lib/ambari-metrics-grafana/public # enable gzip ;enable_gzip = false -# https certs & key file +# https certs & key file ;cert_file = ;cert_key = @@ -145,13 +145,13 @@ static_root_path = /usr/lib/ambari-metrics-grafana/public #################################### Anonymous Auth ########################## [auth.anonymous] # enable anonymous access -;enabled = false +enabled = true # specify organization name that should be used for unauthenticated users -;org_name = Main Org. +org_name = Main Org. # specify role for unauthenticated users -;org_role = Viewer +org_role = Admin #################################### Github Auth ########################## [auth.github] @@ -233,7 +233,7 @@ static_root_path = /usr/lib/ambari-metrics-grafana/public # Max line number of single file, default is 1000000 ;max_lines = 1000000 -# Max size shift of single file, default is 28 means 1 << 28, 256MB +# Max size shift of single file, default is 28 means 1 << 28, 256MB ;max_lines_shift = 28 # Segment log daily, default is true http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java index 8cbc56b..df4fc89 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java @@ -78,6 +78,7 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest { @AfterClass public static void doTeardown() throws Exception { dropNonSystemTables(); + tearDownMiniCluster(); } @Before http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml new file mode 100644 index 0000000..de32ead --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + <property> + <name>metrics_grafana_log_dir</name> + <value>/var/log/ambari-metrics-grafana</value> + <display-name>Metrics Grafana log dir</display-name> + <description>Metrics Grafana log directory.</description> + </property> + <property> + <name>metrics_grafana_pid_dir</name> + <value>/var/run/ambari-metrics-grafana</value> + <display-name>Metrics Grafana pid dir</display-name> + <description>Metrics Grafana pid directory.</description> + </property> + <property> + <name>metrics_grafana_data_dir</name> + <value>/var/lib/ambari-metrics-grafana</value> + <display-name>Metrics Grafana data dir</display-name> + <description>Metrics Grafana data directory.</description> + </property> + <property> + <name>content</name> + <value> +# Set environment variables here. + +# AMS UI Server Home Dir +export AMS_GRAFANA_HOME_DIR={{ams_grafana_home_dir}} + +# AMS UI Server Data Dir +export AMS_GRAFANA_DATA_DIR={{ams_grafana_data_dir}} + +# AMS UI Server Log Dir +export AMS_GRAFANA_LOG_DIR={{ams_grafana_log_dir}} + +# AMS UI Server PID Dir +export AMS_GRAFANA_PID_DIR={{ams_grafana_pid_dir}} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-ini.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-ini.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-ini.xml new file mode 100644 index 0000000..ac1f7fc --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-ini.xml @@ -0,0 +1,282 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + <property> + <name>content</name> + <value> + ##################### Grafana Configuration Example ##################### + # + # Everything has defaults so you only need to uncomment things you want to + # change + + # possible values : production, development + ; app_mode = production + + #################################### Paths #################################### + [paths] + # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) + # + ;data = /var/lib/grafana + data = /var/lib/ambari-metrics-grafana + # + # Directory where grafana can store logs + # + ;logs = /var/log/grafana + logs = /var/log/ambari-metrics-grafana + + + #################################### Server #################################### + [server] + # Protocol (http or https) + ;protocol = http + + # The ip address to bind to, empty will bind to all interfaces + ;http_addr = + + # The http port to use + ;http_port = 3000 + + # The public facing domain name used to access grafana from a browser + ;domain = localhost + + # Redirect to correct domain if host header does not match domain + # Prevents DNS rebinding attacks + ;enforce_domain = false + + # The full public facing url + ;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + + # Log web requests + ;router_logging = false + + # the path relative working path + ;static_root_path = public + static_root_path = /usr/lib/ambari-metrics-grafana/public + + # enable gzip + ;enable_gzip = false + + # https certs & key file + ;cert_file = + ;cert_key = + + #################################### Database #################################### + [database] + # Either "mysql", "postgres" or "sqlite3", it's your choice + ;type = sqlite3 + ;host = 127.0.0.1:3306 + ;name = grafana + ;user = root + ;password = + + # For "postgres" only, either "disable", "require" or "verify-full" + ;ssl_mode = disable + + # For "sqlite3" only, path relative to data_path setting + ;path = grafana.db + + #################################### Session #################################### + [session] + # Either "memory", "file", "redis", "mysql", "postgres", default is "file" + ;provider = file + + # Provider config options + # memory: not have any config yet + # file: session dir path, is relative to grafana data_path + # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` + # mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` + # postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable + ;provider_config = sessions + + # Session cookie name + ;cookie_name = grafana_sess + + # If you use session in https only, default is false + ;cookie_secure = false + + # Session life time, default is 86400 + ;session_life_time = 86400 + + #################################### Analytics #################################### + [analytics] + # Server reporting, sends usage counters to stats.grafana.org every 24 hours. + # No ip addresses are being tracked, only simple counters to track + # running instances, dashboard and error counts. It is very helpful to us. + # Change this option to false to disable reporting. + ;reporting_enabled = true + + # Google Analytics universal tracking code, only enabled if you specify an id here + ;google_analytics_ua_id = + + #################################### Security #################################### + [security] + # default admin user, created on startup + ;admin_user = admin + + # default admin password, can be changed before first start of grafana, or in profile settings + ;admin_password = admin + + # used for signing + ;secret_key = SW2YcwTIb9zpOOhoPsMm + + # Auto-login remember days + ;login_remember_days = 7 + ;cookie_username = grafana_user + ;cookie_remember_name = grafana_remember + + # disable gravatar profile images + ;disable_gravatar = false + + # data source proxy whitelist (ip_or_domain:port seperated by spaces) + ;data_source_proxy_whitelist = + + #################################### Users #################################### + [users] + # disable user signup / registration + ;allow_sign_up = true + + # Allow non admin users to create organizations + ;allow_org_create = true + + # Set to true to automatically assign new users to the default organization (id 1) + ;auto_assign_org = true + + # Default role new users will be automatically assigned (if disabled above is set to true) + ;auto_assign_org_role = Viewer + + # Background text for the user field on the login page + ;login_hint = email or username + + #################################### Anonymous Auth ########################## + [auth.anonymous] + # enable anonymous access + enabled = true + + # specify organization name that should be used for unauthenticated users + org_name = Main Org. + + # specify role for unauthenticated users + org_role = Admin + + #################################### Github Auth ########################## + [auth.github] + ;enabled = false + ;allow_sign_up = false + ;client_id = some_id + ;client_secret = some_secret + ;scopes = user:email,read:org + ;auth_url = https://github.com/login/oauth/authorize + ;token_url = https://github.com/login/oauth/access_token + ;api_url = https://api.github.com/user + ;team_ids = + ;allowed_organizations = + + #################################### Google Auth ########################## + [auth.google] + ;enabled = false + ;allow_sign_up = false + ;client_id = some_client_id + ;client_secret = some_client_secret + ;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email + ;auth_url = https://accounts.google.com/o/oauth2/auth + ;token_url = https://accounts.google.com/o/oauth2/token + ;api_url = https://www.googleapis.com/oauth2/v1/userinfo + ;allowed_domains = + + #################################### Auth Proxy ########################## + [auth.proxy] + ;enabled = false + ;header_name = X-WEBAUTH-USER + ;header_property = username + ;auto_sign_up = true + + #################################### Basic Auth ########################## + [auth.basic] + ;enabled = true + + #################################### Auth LDAP ########################## + [auth.ldap] + ;enabled = false + ;config_file = /etc/grafana/ldap.toml + + #################################### SMTP / Emailing ########################## + [smtp] + ;enabled = false + ;host = localhost:25 + ;user = + ;password = + ;cert_file = + ;key_file = + ;skip_verify = false + ;from_address = admin@grafana.localhost + + [emails] + ;welcome_email_on_sign_up = false + + #################################### Logging ########################## + [log] + # Either "console", "file", default is "console" + # Use comma to separate multiple modes, e.g. "console, file" + ;mode = console, file + + # Buffer length of channel, keep it as it is if you don't know what it is. + ;buffer_len = 10000 + + # Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" + ;level = Info + + # For "console" mode only + [log.console] + ;level = + + # For "file" mode only + [log.file] + ;level = + # This enables automated log rotate(switch of following options), default is true + ;log_rotate = true + + # Max line number of single file, default is 1000000 + ;max_lines = 1000000 + + # Max size shift of single file, default is 28 means 1 << 28, 256MB + ;max_lines_shift = 28 + + # Segment log daily, default is true + ;daily_rotate = true + + # Expired days of log file(delete after max days), default is 7 + ;max_days = 7 + + #################################### AMPQ Event Publisher ########################## + [event_publisher] + ;enabled = false + ;rabbitmq_url = amqp://localhost/ + ;exchange = grafana_events + + ;#################################### Dashboard JSON files ########################## + [dashboards.json] + ;enabled = false + ;path = /var/lib/grafana/dashboards + path = /usr/lib/ambari-metrics-grafana/public/dashboards + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index 48469df..6ea4ca5 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -62,6 +62,18 @@ <timeout>1200</timeout> </commandScript> </component> + <component> + <name>METRICS_GRAFANA</name> + <displayName>Grafana</displayName> + <category>MASTER</category> + <cardinality>0-1</cardinality> + <versionAdvertised>false</versionAdvertised> + <commandScript> + <script>scripts/metrics_grafana.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </component> </components> <osSpecifics> @@ -81,6 +93,10 @@ <skipUpgrade>true</skipUpgrade> </package> <package> + <name>ambari-metrics-grafana</name> + <skipUpgrade>true</skipUpgrade> + </package> + <package> <name>gcc</name> </package> </packages> @@ -135,6 +151,8 @@ <config-type>ams-hbase-security-site</config-type> <config-type>ams-hbase-env</config-type> <config-type>ams-hbase-log4j</config-type> + <config-type>ams-grafana-env</config-type> + <config-type>ams-grafana-ini</config-type> </configuration-dependencies> <excluded-config-types> http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index 8fdf4ba..eed6063 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -384,5 +384,35 @@ def ams(name=None): # TODO pass + elif name == 'grafana': + + ams_grafana_directories = [ + params.ams_grafana_conf_dir, + params.ams_grafana_log_dir, + params.ams_grafana_data_dir, + params.ams_grafana_pid_dir + ] + + for ams_grafana_directory in ams_grafana_directories: + Directory(ams_grafana_directory, + owner=params.ams_user, + group=params.user_group, + mode=0755, + recursive=True + ) + + File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"), + owner=params.ams_user, + group=params.user_group, + content=InlineTemplate(params.ams_grafana_env_sh_template) + ) + + File(format("{ams_grafana_conf_dir}/ams-grafana.ini"), + owner=params.ams_user, + group=params.user_group, + content=InlineTemplate(params.ams_grafana_ini_template) + ) + + pass pass http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py index cf498ec..ddefed7 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py @@ -31,7 +31,7 @@ from ambari_commons.os_family_impl import OsFamilyImpl class AmsCollector(Script): def install(self, env): - self.install_packages(env) + self.install_packages(env, exclude_packages = ['ambari-metrics-grafana']) def configure(self, env, action = None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py new file mode 100644 index 0000000..f876036 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import Script, Execute +from resource_management.libraries.functions import format +from status import check_service_status +from ams import ams + +class AmsGrafana(Script): + def install(self, env): + self.install_packages(env, exclude_packages = ['ambari-metrics-collector']) + + def configure(self, env, action = None): + import params + env.set_params(params) + ams(name='grafana') + + def start(self, env): + import params + env.set_params(params) + self.configure(env, action = 'start') + + stop_cmd = format("{ams_grafana_script} stop") + Execute(stop_cmd, + user=params.ams_user + ) + + start_cmd = format("{ams_grafana_script} start") + Execute(start_cmd, + user=params.ams_user + ) + + def stop(self, env): + import params + env.set_params(params) + self.configure(env, action = 'stop') + stop_cmd = format("{ams_grafana_script} stop") + Execute(stop_cmd, + user=params.ams_user + ) + + def status(self, env): + import status_params + env.set_params(status_params) + check_service_status(name='grafana') + +if __name__ == "__main__": + AmsGrafana().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py index a1d6a1d..f392984 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py @@ -25,7 +25,7 @@ from status import check_service_status class AmsMonitor(Script): def install(self, env): - self.install_packages(env, exclude_packages = ['ambari-metrics-collector']) + self.install_packages(env, exclude_packages = ['ambari-metrics-collector', 'ambari-metrics-grafana']) self.configure(env) # for security def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index a1de919..d8a19ce 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -72,6 +72,13 @@ ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring" ams_monitor_pid_dir = status_params.ams_monitor_pid_dir ams_monitor_script = "/usr/sbin/ambari-metrics-monitor" +ams_grafana_script = "/usr/sbin/ambari-metrics-grafana" +ams_grafana_home_dir = '/usr/lib/ambari-metrics-grafana' +ams_grafana_log_dir = default("/configurations/ams-grafana-env/metrics_grafana_log_dir", '/var/log/ambari-metrics-grafana') +ams_grafana_pid_dir = status_params.ams_grafana_pid_dir +ams_grafana_conf_dir = '/etc/ambari-metrics-grafana/conf' +ams_grafana_data_dir = default("/configurations/ams-grafana-env/metrics_grafana_data_dir", '/var/lib/ambari-metrics-grafana') + ams_hbase_home_dir = "/usr/lib/ams-hbase/" ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None) @@ -220,6 +227,8 @@ else: hbase_env_sh_template = config['configurations']['ams-hbase-env']['content'] ams_env_sh_template = config['configurations']['ams-env']['content'] +ams_grafana_env_sh_template = config['configurations']['ams-grafana-env']['content'] +ams_grafana_ini_template = config['configurations']['ams-grafana-ini']['content'] hbase_staging_dir = default("/configurations/ams-hbase-site/hbase.bulkload.staging.dir", "/amshbase/staging") http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status.py index 59466ad..fa4e8a4 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status.py @@ -37,6 +37,10 @@ def check_service_status(name): pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid") check_process_status(pid_file) + elif name == 'grafana': + pid_file = format("{ams_grafana_pid_dir}/grafana-server.pid") + check_process_status(pid_file) + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def check_service_status(name): import service_mapping http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py index d446baa..3c4bffc 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py @@ -30,6 +30,7 @@ hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir'] hbase_user = ams_user ams_collector_pid_dir = config['configurations']['ams-env']['metrics_collector_pid_dir'] ams_monitor_pid_dir = config['configurations']['ams-env']['metrics_monitor_pid_dir'] +ams_grafana_pid_dir = config['configurations']['ams-grafana-env']['metrics_grafana_pid_dir'] security_enabled = config['configurations']['cluster-env']['security_enabled'] ams_hbase_conf_dir = format("{hbase_conf_dir}") http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json index a480e2a..710b0dd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json @@ -51,7 +51,9 @@ "NAMENODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP", "METRICS_COLLECTOR-STOP"], "DATANODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", - "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"] + "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "METRICS_GRAFANA-START": ["METRICS_COLLECTOR-START"], + "METRICS_COLLECTOR-STOP": ["METRICS_GRAFANA-STOP"] }, "_comment" : "Dependencies that are used in HA NameNode cluster", "namenode_optional_ha": { http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py new file mode 100644 index 0000000..532a4b8 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from mock.mock import MagicMock, patch +from stacks.utils.RMFTestCase import * + +@patch("os.path.exists", new = MagicMock(return_value=True)) +@patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) +class TestMetricsGrafana(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package" + STACK_VERSION = "2.0.6" + + def test_start(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metrics_grafana.py", + classname = "AmsGrafana", + command = "start", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.maxDiff=None + self.assert_configure() + self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-grafana stop', + user = 'ams' + ) + self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-grafana start', + user = 'ams' + ) + self.assertNoMoreResources() + + def assert_configure(self): + + ams_grafana_directories = [ + '/etc/ambari-metrics-grafana/conf', + '/var/log/ambari-metrics-grafana', + '/var/lib/ambari-metrics-grafana', + '/var/run/ambari-metrics-grafana' + ] + + for ams_grafana_directory in ams_grafana_directories: + self.assertResourceCalled('Directory', ams_grafana_directory, + owner = 'ams', + group = 'hadoop', + mode=0755, + recursive = True + ) + + self.assertResourceCalled('File', '/etc/ambari-metrics-grafana/conf/ams-grafana-env.sh', + owner = 'ams', + group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['ams-env']['content']) + ) + + self.assertResourceCalled('File', '/etc/ambari-metrics-grafana/conf/ams-grafana.ini', + owner = 'ams', + group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['ams-env']['content']) + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/d8920559/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index f421aec..b139190 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -852,6 +852,15 @@ "timeline.metrics.daily.aggregator.minute.interval": "86400", "timeline.metrics.cluster.aggregator.minute.interval": "120", "timeline.metrics.host.aggregator.hourly.interval": "3600" + }, + "ams-grafana-env": { + "metrics_grafana_log_dir": "/var/log/ambari-metrics-grafana", + "metrics_grafana_pid_dir": "/var/run/ambari-metrics-grafana", + "metrics_grafana_data_dir": "/var/lib/ambari-metrics-grafana", + "content": "\n" + }, + "ams-grafana-ini": { + "content": "\n" } }, "configuration_attributes": {