AMBARI-20802. StackAdvisor should not recommend configuration types for services that are not present in the cluster.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b27feb7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b27feb7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b27feb7 Branch: refs/heads/branch-3.0-perf Commit: 4b27feb7fdc3adcefed6c89dd5ba9c357c5b6944 Parents: 2a00812 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Fri Apr 21 18:15:30 2017 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Mon Apr 24 14:59:56 2017 +0300 ---------------------------------------------------------------------- .../ambari/server/topology/AmbariContext.java | 3 + .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 1 + .../HIVE/0.12.0.2.0/package/scripts/hive.py | 25 +++++--- .../stacks/HDP/2.1/services/stack_advisor.py | 11 ++-- .../stacks/HDP/2.2/services/stack_advisor.py | 61 ++++++++++---------- .../stacks/2.1/HIVE/test_hive_metastore.py | 7 +++ .../stacks/2.1/common/test_stack_advisor.py | 7 ++- .../stacks/2.6/common/test_stack_advisor.py | 6 +- 8 files changed, 76 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 6d12402..5e0d707 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -427,6 +427,9 @@ public class AmbariContext { for (String actualConfigType : updatedConfigTypes) { // get the actual cluster config for comparison DesiredConfig actualConfig = cluster.getDesiredConfigs().get(actualConfigType); + if (actualConfig == null && actualConfigType.equals("core-site")) { + continue; + } if (!actualConfig.getTag().equals(TopologyManager.TOPOLOGY_RESOLVED_TAG)) { // if any expected config is not resolved, deployment must wait LOG.info("Config type " + actualConfigType + " not resolved yet, Blueprint deployment will wait until configuration update is completed"); http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml index 5f56617..caa598a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml @@ -220,6 +220,7 @@ <display-name>hive-env template</display-name> <description>This is the jinja template for hive-env.sh file</description> <value> + export JAVA_HOME={{java64_home}} if [ "$SERVICE" = "cli" ]; then if [ -z "$DEBUG" ]; then export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index b7b04a2..959e111 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -300,6 +300,15 @@ def setup_metastore(): content=StaticFile('startMetastore.sh') ) + if not is_empty(params.hive_exec_scratchdir): + dirPathStr = urlparse(params.hive_exec_scratchdir).path + pathComponents = dirPathStr.split("/") + if dirPathStr.startswith("/tmp") and len(pathComponents) > 2: + Directory (params.hive_exec_scratchdir, + owner = params.hive_user, + create_parents = True, + mode=0777) + def create_metastore_schema(): import params @@ -345,14 +354,14 @@ def fill_conf_dir(component_conf_dir): mode=mode_identified_for_dir ) - XmlConfig("mapred-site.xml", - conf_dir=component_conf_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.hive_user, - group=params.user_group, - mode=mode_identified_for_file) - + if 'mapred-site' in params.config['configurations']: + XmlConfig("mapred-site.xml", + conf_dir=component_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.hive_user, + group=params.user_group, + mode=mode_identified_for_file) File(format("{component_conf_dir}/hive-default.xml.template"), owner=params.hive_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index 866d4cb..ddd795f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -210,12 +210,13 @@ class HDP21StackAdvisor(HDP206StackAdvisor): webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true') webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true') - if self.is_secured_cluster(services): - putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + if "HDFS" in servicesList or "YARN" in servicesList: + if self.is_secured_cluster(services): + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) - meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services) - if "hostnames" in meta: - self.put_proxyuser_value("HTTP", meta["hostnames"], services=services, configurations=configurations, put_function=putCoreSiteProperty) + meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services) + if "hostnames" in meta: + self.put_proxyuser_value("HTTP", meta["hostnames"], services=services, configurations=configurations, put_function=putCoreSiteProperty) def recommendTezConfigurations(self, configurations, clusterData, services, hosts): putTezProperty = self.putProperty(configurations, "tez-site") http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 4d672d2..726514b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -422,36 +422,37 @@ class HDP22StackAdvisor(HDP21StackAdvisor): container_size = "512" - if not "yarn-site" in configurations: - self.recommendYARNConfigurations(configurations, clusterData, services, hosts) - #properties below should be always present as they are provided in HDP206 stack advisor at least - yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - #duplicate tez task resource calc logic, direct dependency doesn't look good here (in case of Hive without Tez) - container_size = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) - container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) - - putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) - - putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - - if "yarn-site" in services["configurations"]: - if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: - putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: - putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - - putHiveSiteProperty("hive.prewarm.enabled", "false") - putHiveSiteProperty("hive.prewarm.numcontainers", "3") - putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true") - putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true") - - container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"] - container_size_bytes = int(int(container_size)*0.8*1024*1024) # Xmx == 80% of container - # Memory - putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3))) - putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum", container_size_bytes) - putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") + if "YARN" in servicesList: + if not "yarn-site" in configurations: + self.recommendYARNConfigurations(configurations, clusterData, services, hosts) + #properties below should be always present as they are provided in HDP206 stack advisor at least + yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + #duplicate tez task resource calc logic, direct dependency doesn't look good here (in case of Hive without Tez) + container_size = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) + container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) + + putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) + + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + + if "yarn-site" in services["configurations"]: + if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + + putHiveSiteProperty("hive.prewarm.enabled", "false") + putHiveSiteProperty("hive.prewarm.numcontainers", "3") + putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true") + putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true") + + container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"] + container_size_bytes = int(int(container_size)*0.8*1024*1024) # Xmx == 80% of container + # Memory + putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3))) + putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum", container_size_bytes) + putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 94f639a..e34734c 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -688,6 +688,13 @@ class TestHiveMetastore(RMFTestCase): content = StaticFile('startMetastore.sh'), mode = 0755, ) + + self.assertResourceCalled('Directory', '/tmp/hive', + owner = 'hive', + create_parents = True, + mode=0777 + ) + self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/usr/share/java/mysql-connector-java.jar', http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py index e8bd5d0..bdd34fc 100644 --- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py @@ -272,6 +272,11 @@ class TestHDP21StackAdvisor(TestCase): "services": [ { "StackServices": { + "service_name": "YARN" + }, "components": [] + }, + { + "StackServices": { "service_name": "HIVE", }, "components": [ @@ -338,7 +343,7 @@ class TestHDP21StackAdvisor(TestCase): self.assertEqual(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] == "example.com", True) newhost_list = ["example.com", "example.org"] - services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = newhost_list + services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = newhost_list configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] = "" self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b27feb7/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index 2d7322d..114dd3a 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -1012,7 +1012,11 @@ class TestHDP26StackAdvisor(TestCase): services = { "services": - [ + [{ + "StackServices": { + "service_name": "YARN" + }, "components": [] + }, { "StackServices": { "service_name" : "HIVE",