Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-18901 24045aa89 -> b74905495
AMBARI-19035. LLAP startup: Ambari should not use slider anti-affinity when it's avoidable Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7490549 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7490549 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7490549 Branch: refs/heads/branch-feature-AMBARI-18901 Commit: b74905495babf00dc93dd24e28290aea3c7acb3d Parents: 24045aa Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Wed Nov 30 11:24:49 2016 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Wed Nov 30 16:02:14 2016 -0800 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 12 +++++++++++- .../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b7490549/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 00e5fc1..4866432 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -277,10 +277,20 @@ class HiveServerInteractiveDefault(HiveServerInteractive): unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') + # Figure out the Slider Anti-affinity to be used. + # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it + # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP + # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. + slider_placement = 4 + if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): + slider_placement = 0 + Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " + "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m " " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") + " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) http://git-wip-us.apache.org/repos/asf/ambari/blob/b7490549/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index bba0f3b..e7461fd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -599,6 +599,7 @@ if has_hive_interactive: tez_interactive_user = config['configurations']['tez-env']['tez_user'] num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10) # Used in LLAP slider package creation + yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb'] num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes'] llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb'] llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']