Repository: ambari Updated Branches: refs/heads/branch-2.2 d59600e62 -> 246dc6800
AMBARI-14165. Populate hawq's yarn-client xml under /usr/local/hawq/etc (bhuvnesh chaudhary via odiachenko). Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/246dc680 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/246dc680 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/246dc680 Branch: refs/heads/branch-2.2 Commit: 246dc6800b95baf45df93cba205889eca1c38c9e Parents: d59600e Author: Oleksandr Diachenko <odiache...@pivotal.io> Authored: Thu Dec 24 12:12:57 2015 -0800 Committer: Oleksandr Diachenko <odiache...@pivotal.io> Committed: Thu Dec 24 12:12:57 2015 -0800 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/yarn-client.xml | 99 ++++++++++++++++++++ .../HAWQ/2.0.0/package/scripts/common.py | 55 +++++++++++ .../HAWQ/2.0.0/package/scripts/constants.py | 1 + .../HAWQ/2.0.0/package/scripts/params.py | 4 +- 4 files changed, 157 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml new file mode 100755 index 0000000..a2296a1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml @@ -0,0 +1,99 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <!-- RPC client configuration --> + <property> + <name>rpc.client.timeout</name> + <value>3600000</value> + <description> + Timeout interval of a RPC invocation in millisecond. Default is 3600000. + </description> + </property> + <property> + <name>rpc.client.connect.tcpnodelay</name> + <value>true</value> + <description> + Whether to set socket TCP_NODELAY to true when connecting to RPC server. Default is true. + </description> + </property> + + <property> + <name>rpc.client.max.idle</name> + <value>10000</value> + <description> + Max idle time of a RPC connection in milliseconds. Default is 10000. + </description> + </property> + + <property> + <name>rpc.client.ping.interval</name> + <value>10000</value> + <description> + Interval at which the RPC client sends a heart beat to server. 0 means disable, default is 10000. + </description> + </property> + + <property> + <name>rpc.client.connect.timeout</name> + <value>600000</value> + <description> + Timeout interval in milliseconds for which RPC client tries to setup the connection. Default is 600000. + </description> + </property> + + <property> + <name>rpc.client.connect.retry</name> + <value>10</value> + <description> + Max retry attempts if the RPC client fails to setup the connection to server. Default is 10. + </description> + </property> + + <property> + <name>rpc.client.read.timeout</name> + <value>3600000</value> + <description> + Timeout interval in milliseconds when the RPC client is trying to read from server. Default is 3600000. + </description> + </property> + + <property> + <name>rpc.client.write.timeout</name> + <value>3600000</value> + <description> + Timeout interval in milliseconds when the RPC client is trying to write to server. Default is 3600000. + </description> + </property> + + <property> + <name>rpc.client.socket.linger.timeout</name> + <value>-1</value> + <description> + Set value to socket SO_LINGER when connecting to RPC server. -1 means default OS value. Default is -1. + </description> + </property> + + <property> + <name>yarn.client.failover.max.attempts</name> + <value>15</value> + <description> + If multiple resource managers are configured, it is the max retry attempts for which yarn client tries to issue a RPC call. Default is 15. + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py index 31ef2fa..c62ac12 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py @@ -22,6 +22,7 @@ import crypt import filecmp from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.core.resources.system import Execute, Directory, File +from resource_management.libraries.script.config_dictionary import ConfigDictionary from resource_management.core.logger import Logger from resource_management.core.system import System from resource_management.core.exceptions import Fail @@ -88,9 +89,63 @@ def setup_common_configurations(): owner=constants.hawq_user, group=constants.hawq_group, mode=0644) + if "yarn-site" in params.config["configurations"]: + __update_yarn_client() __set_osparams() +def __update_yarn_client(): + """ + Writes yarn-client.xml on the local filesystem on hawq nodes. + If yarn ha is enabled, appends related parameters to yarn-client.xml + """ + import params + + yarn_client_dict = params.yarn_client.copy() + if params.yarn_ha_enabled: + # Temporary logic, this logic will be moved in ambari-web to expose these parameters on UI once Yarn HA is enabled + rm_ids = [rm_id.strip() for rm_id in params.config['configurations']['yarn-site']['yarn.resourcemanager.ha.rm-ids'].split(',')] + rm_id1 = rm_ids[0] + rm_id2 = rm_ids[1] + # Identify the hostname for yarn resource managers + rm_host1= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id1)] + rm_host2= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id2)] + # Ambari does not update yarn.resourcemanager.address.${rm_id} and yarn.resourcemanager.scheduler.address.${rm_id} + # property as its derived automatically at yarn. + # Hawq uses these properties to use yarn ha. If these properties are defined at Ambari use them, else derive them. + # Use port 8032 to derive hawq.resourcemanager.address.${rm_id}:port value if needed + rm_default_port = 8032 + # Use port 8030 to derive hawq.resourcemanager.scheduler.address.${rm_id}:port value if needed + rm_scheduler_default_port = 8030 + + rm_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id1)) + if rm_address_host1 is None: + rm_address_host1 = "{0}:{1}".format(rm_host1, rm_default_port) + + rm_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id2)) + if rm_address_host2 is None: + rm_address_host2 = "{0}:{1}".format(rm_host2, rm_default_port) + + rm_scheduler_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id1)) + if rm_scheduler_address_host1 is None: + rm_scheduler_address_host1 = "{0}:{1}".format(rm_host1, rm_scheduler_default_port) + + rm_scheduler_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id2)) + if rm_scheduler_address_host2 is None: + rm_scheduler_address_host2 = "{0}:{1}".format(rm_host2, rm_scheduler_default_port) + + yarn_client_dict['yarn.resourcemanager.ha'] = "{0},{1}".format(rm_address_host1, rm_address_host2) + yarn_client_dict['yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(rm_scheduler_address_host1, rm_scheduler_address_host2) + + XmlConfig("yarn-client.xml", + conf_dir=constants.hawq_config_dir, + configurations=ConfigDictionary(yarn_client_dict), + configuration_attributes=params.config['configuration_attributes']['yarn-client'], + owner=constants.hawq_user, + group=constants.hawq_group, + mode=0644) + + def __substitute_hostnames_in_hawq_site(): """ Temporary function to replace localhost with actual HAWQ component hostnames. http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py index b56fd59..ff8f4be 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py @@ -23,6 +23,7 @@ SEGMENT = "segment" START = "start" INIT = "init" STOP = "stop" +YARN = "yarn" # Users root_user = "root" http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py index fb449b9..c8e97b8 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py @@ -70,6 +70,7 @@ namenode_host= __get_namenode_host() # YARN # Note: YARN is not mandatory for HAWQ. It is required only when the users set HAWQ to use YARN as resource manager rm_host = __get_component_host('rm_host') +yarn_ha_enabled = default('/configurations/yarn-site/yarn.resourcemanager.ha.enabled', False) # Config files gpcheck_content = config['configurations']['gpcheck-env']['content'] @@ -79,6 +80,7 @@ hawq_limits = config['configurations']['hawq-limits-env'] hawq_sysctl = config['configurations']['hawq-sysctl-env'] hawq_site = config['configurations']['hawq-site'] +yarn_client = config['configurations']['yarn-client'] hawq_master_dir = hawq_site.get('hawq_master_directory') hawq_segment_dir = hawq_site.get('hawq_segment_directory') hawq_master_temp_dir = hawq_site.get('hawq_master_temp_directory') @@ -88,5 +90,3 @@ hawq_segment_temp_dir = hawq_site.get('hawq_segment_temp_directory') hawq_hdfs_data_dir = "/{0}".format(hawq_site.get('hawq_dfs_url').split('/', 1)[1]) hawq_master_address_port = hawq_site.get('hawq_master_address_port') hawq_segment_address_port = hawq_site.get('hawq_segment_address_port') - -