This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push: new c22b8e7 AMBARI-24837. Make Grafana connection attempts and retry delay configurable (#2525) c22b8e7 is described below commit c22b8e7cafdff214079653e8cbef1fd6baa467b4 Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Mon Oct 29 13:43:18 2018 +0100 AMBARI-24837. Make Grafana connection attempts and retry delay configurable (#2525) --- .../0.1.0/configuration/ams-grafana-env.xml | 22 +++++++++ .../0.1.0/package/scripts/metrics_grafana.py | 3 +- .../0.1.0/package/scripts/metrics_grafana_util.py | 54 +++++++++++----------- .../AMBARI_METRICS/0.1.0/package/scripts/params.py | 3 ++ 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml index eaafc6b..b60d3d9 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-grafana-env.xml @@ -90,4 +90,26 @@ export AMS_GRAFANA_PID_DIR={{ams_grafana_pid_dir}} </value> <on-ambari-upgrade add="true"/> </property> + <property> + <name>metrics_grafana_connect_retry_delay</name> + <value>20</value> + <display-name>Grafana connect retry delay</display-name> + <description>The time in seconds after which connection to Grafana is retried in case of failure</description> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>metrics_grafana_connect_attempts</name> + <value>15</value> + <display-name>Grafana connect attempts</display-name> + <description>The number of attempts for connection to Grafana</description> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> </configuration> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py index 7fe168b..8b696f4 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py @@ -22,7 +22,6 @@ from resource_management import Script, Execute from resource_management.libraries.functions import format from status import check_service_status from ams import ams -from metrics_grafana_util import create_ams_datasource, create_ams_dashboards, create_grafana_admin_pwd from resource_management.core.logger import Logger from resource_management.core import sudo @@ -54,6 +53,8 @@ class AmsGrafana(Script): else: Logger.info("Grafana Server has started with pid: {0}".format(sudo.read_file(pidfile).strip())) + from metrics_grafana_util import create_ams_datasource, create_ams_dashboards, create_grafana_admin_pwd + #Set Grafana admin pwd create_grafana_admin_pwd() # Create datasource diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py index 25c12cf..fe33aa3 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py @@ -35,8 +35,6 @@ import ambari_simplejson as json import ambari_commons.network as network import os -GRAFANA_CONNECT_TRIES = 15 -GRAFANA_CONNECT_TIMEOUT = 20 GRAFANA_SEARCH_BUILTIN_DASHBOARDS = "/api/search?tag=builtin" GRAFANA_DATASOURCE_URL = "/api/datasources" GRAFANA_USER_URL = "/api/user" @@ -46,14 +44,15 @@ METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS" Server = namedtuple('Server', [ 'protocol', 'host', 'port', 'user', 'password' ]) def perform_grafana_get_call(url, server): + import params + grafana_https_enabled = server.protocol.lower() == 'https' response = None ca_certs = None if grafana_https_enabled: - import params ca_certs = params.ams_grafana_ca_cert - for i in xrange(0, GRAFANA_CONNECT_TRIES): + for i in xrange(0, params.grafana_connect_attempts): try: conn = network.get_http_connection( server.host, @@ -73,10 +72,10 @@ def perform_grafana_get_call(url, server): Logger.info("Http response: %s %s" % (response.status, response.reason)) break except (httplib.HTTPException, socket.error) as ex: - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) + if i < params.grafana_connect_attempts - 1: Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) + % (params.grafana_connect_retry_delay)) + time.sleep(params.grafana_connect_retry_delay) continue else: raise Fail("Ambari Metrics Grafana update failed due to: %s" % str(ex)) @@ -85,6 +84,8 @@ def perform_grafana_get_call(url, server): return response def perform_grafana_put_call(url, id, payload, server): + import params + response = None data = None userAndPass = b64encode('{0}:{1}'.format(server.user, server.password)) @@ -94,10 +95,9 @@ def perform_grafana_put_call(url, id, payload, server): ca_certs = None if grafana_https_enabled: - import params ca_certs = params.ams_grafana_ca_cert - for i in xrange(0, GRAFANA_CONNECT_TRIES): + for i in xrange(0, params.grafana_connect_attempts): try: conn = network.get_http_connection( server.host, @@ -113,10 +113,10 @@ def perform_grafana_put_call(url, id, payload, server): conn.close() break except (httplib.HTTPException, socket.error) as ex: - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) + if i < params.grafana_connect_attempts - 1: Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) + % (params.grafana_connect_retry_delay)) + time.sleep(params.grafana_connect_retry_delay) continue else: raise Fail("Ambari Metrics Grafana update failed due to: %s" % str(ex)) @@ -125,6 +125,8 @@ def perform_grafana_put_call(url, id, payload, server): return (response, data) def perform_grafana_post_call(url, payload, server): + import params + response = None data = None userAndPass = b64encode('{0}:{1}'.format(server.user, server.password)) @@ -135,10 +137,9 @@ def perform_grafana_post_call(url, payload, server): ca_certs = None if grafana_https_enabled: - import params ca_certs = params.ams_grafana_ca_cert - for i in xrange(0, GRAFANA_CONNECT_TRIES): + for i in xrange(0, params.grafana_connect_attempts): try: Logger.info("Connecting (POST) to %s:%s%s" % (server.host, server.port, url)) conn = network.get_http_connection( @@ -147,26 +148,26 @@ def perform_grafana_post_call(url, payload, server): grafana_https_enabled, ca_certs, ssl_version=Script.get_force_https_protocol_value() ) - + conn.request("POST", url, payload, headers) response = conn.getresponse() Logger.info("Http response: %s %s" % (response.status, response.reason)) if response.status == 401: #Intermittent error thrown from Grafana - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) + if i < params.grafana_connect_attempts - 1: Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) + % (params.grafana_connect_retry_delay)) + time.sleep(params.grafana_connect_retry_delay) continue data = response.read() Logger.info("Http data: %s" % data) conn.close() break except (httplib.HTTPException, socket.error) as ex: - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) + if i < params.grafana_connect_attempts - 1: Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) + % (params.grafana_connect_retry_delay)) + time.sleep(params.grafana_connect_retry_delay) continue else: raise Fail("Ambari Metrics Grafana update failed due to: %s" % str(ex)) @@ -175,15 +176,16 @@ def perform_grafana_post_call(url, payload, server): return (response, data) def perform_grafana_delete_call(url, server): + import params + grafana_https_enabled = server.protocol.lower() == 'https' response = None ca_certs = None if grafana_https_enabled: - import params ca_certs = params.ams_grafana_ca_cert - for i in xrange(0, GRAFANA_CONNECT_TRIES): + for i in xrange(0, params.grafana_connect_attempts): try: conn = network.get_http_connection( server.host, @@ -202,10 +204,10 @@ def perform_grafana_delete_call(url, server): Logger.info("Http response: %s %s" % (response.status, response.reason)) break except (httplib.HTTPException, socket.error) as ex: - if i < GRAFANA_CONNECT_TRIES - 1: - time.sleep(GRAFANA_CONNECT_TIMEOUT) + if i < params.grafana_connect_attempts - 1: Logger.info("Connection to Grafana failed. Next retry in %s seconds." - % (GRAFANA_CONNECT_TIMEOUT)) + % (params.grafana_connect_retry_delay)) + time.sleep(params.grafana_connect_retry_delay) continue else: raise Fail("Ambari Metrics Grafana update failed due to: %s" % str(ex)) diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index f7fd545..dd03627 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -177,6 +177,9 @@ ams_grafana_cert_file = default("/configurations/ams-grafana-ini/cert_file", '/e ams_grafana_cert_key = default("/configurations/ams-grafana-ini/cert_key", '/etc/ambari-metrics/conf/ams-grafana.key') ams_grafana_ca_cert = default("/configurations/ams-grafana-ini/ca_cert", None) +grafana_connect_attempts = max(int(default('/configurations/ams-grafana-env/metrics_grafana_connect_attempts', 15)), 1) +grafana_connect_retry_delay = max(int(default('/configurations/ams-grafana-env/metrics_grafana_connect_retry_delay', 20)), 1) + ams_hbase_home_dir = "/usr/lib/ams-hbase/" ams_hbase_init_check_enabled = default("/configurations/ams-site/timeline.metrics.hbase.init.check.enabled", True)