AMBARI-20553 : Ambari script error for ams-hbase while writing to Amazon s3 on a cluster with no HDFS. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a78a93f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a78a93f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a78a93f Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 5a78a93fba4336c128b1315bc4463f61e6f6c981 Parents: f44d9d2 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Wed Mar 29 14:13:42 2017 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Wed Mar 29 14:13:42 2017 -0700 ---------------------------------------------------------------------- .../conf/unix/ambari-metrics-collector | 3 ++- .../AMBARI_METRICS/0.1.0/configuration/ams-env.xml | 11 +++++++++++ .../AMBARI_METRICS/0.1.0/package/scripts/hbase.py | 11 ++++++++++- .../AMBARI_METRICS/0.1.0/package/scripts/params.py | 4 ++++ 4 files changed, 27 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5a78a93f/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector index c106235..552be48 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector @@ -38,6 +38,7 @@ METRIC_COLLECTOR=ambari-metrics-collector NORMALIZER_ENABLED_STUB_FILE=/var/run/ambari-metrics-collector/normalizer_enabled FIFO_ENABLED_STUB_FILE=/var/run/ambari-metrics-collector/fifo_enabled +COLLECTOR_ADDITIONAL_CLASSPATH= STOP_TIMEOUT=5 @@ -256,7 +257,7 @@ function start() rm -f "${PIDFILE}" >/dev/null 2>&1 fi - nohup "${JAVA}" "-Xms$AMS_COLLECTOR_HEAPSIZE" "-Xmx$AMS_COLLECTOR_HEAPSIZE" ${AMS_COLLECTOR_OPTS} "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dams.log.dir=${AMS_COLLECTOR_LOG_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 & + nohup "${JAVA}" "-Xms$AMS_COLLECTOR_HEAPSIZE" "-Xmx$AMS_COLLECTOR_HEAPSIZE" ${AMS_COLLECTOR_OPTS} "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}:${COLLECTOR_ADDITIONAL_CLASSPATH}" "-Djava.net.preferIPv4Stack=true" "-Dams.log.dir=${AMS_COLLECTOR_LOG_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 & PID=$! write_pidfile "${PIDFILE}" sleep 2 http://git-wip-us.apache.org/repos/asf/ambari/blob/5a78a93f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml index d79e357..cb66537 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml @@ -117,6 +117,15 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>ams_classpath_additional</name> + <value/> + <description>Additional directory or jars in classpath for Metrics Collector.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>content</name> <display-name>ams-env template</display-name> <value> @@ -162,6 +171,8 @@ export AMS_COLLECTOR_OPTS="$AMS_COLLECTOR_OPTS $AMS_COLLECTOR_GC_OPTS" # Metrics collector host will be blacklisted for specified number of seconds if metric monitor failed to connect to it. export AMS_FAILOVER_STRATEGY_BLACKLISTED_INTERVAL={{failover_strategy_blacklisted_interval}} +# Extra Java CLASSPATH elements for Metrics Collector. Optional. +export COLLECTOR_ADDITIONAL_CLASSPATH={{ams_classpath_additional}} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/5a78a93f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py index a5ff4f0..e20ff1d 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py @@ -117,6 +117,15 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' create_parents = True ) + if params.hbase_wal_dir: + Directory(params.hbase_wal_dir, + owner=params.hbase_user, + group = params.user_group, + cd_access="a", + create_parents = True, + recursive_ownership = True, + ) + merged_ams_hbase_site = {} merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site']) if params.security_enabled: @@ -207,7 +216,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' if not params.is_local_fs_rootdir: # If executing Stop All, HDFS is probably down - if action != 'stop': + if action != 'stop' and not params.skip_create_hbase_root_dir: params.HdfsResource(params.hbase_root_dir, type="directory", http://git-wip-us.apache.org/repos/asf/ambari/blob/5a78a93f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index 5672ee5..1733b19 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -336,6 +336,10 @@ ams_grafana_env_sh_template = config['configurations']['ams-grafana-env']['conte ams_grafana_ini_template = config['configurations']['ams-grafana-ini']['content'] hbase_staging_dir = default("/configurations/ams-hbase-site/hbase.bulkload.staging.dir", "/amshbase/staging") +skip_create_hbase_root_dir = default("/configurations/ams-site/timeline.metrics.skip.create.hbase.root.dir", False) +hbase_wal_dir = default("/configurations/ams-hbase-site/hbase.wal.dir", None) +if hbase_wal_dir and re.search("^file://|/", hbase_wal_dir): #If wal dir is on local file system, create it. + hbase_wal_dir = re.sub("^file://|/", "", hbase_wal_dir, count=1) #for create_hdfs_directory hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
