AMBARI-21471. ATS going down due to missing org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/853a5d4a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/853a5d4a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/853a5d4a Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 853a5d4a2eda1afb5ee4578cf99d0757abc5f95d Parents: eb1adcb Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Jul 13 22:35:28 2017 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Jul 13 22:38:40 2017 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.5/services/stack_advisor.py | 5 +- .../src/main/resources/stacks/stack_advisor.py | 19 ++- .../stacks/2.5/common/test_stack_advisor.py | 150 +++++++++++-------- 3 files changed, 105 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 3337e8e..4ca74ee 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -19,6 +19,7 @@ limitations under the License. import math + from ambari_commons.str_utils import string_set_equals from resource_management.core.exceptions import Fail from resource_management.libraries.functions.get_bare_principal import get_bare_principal @@ -774,9 +775,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME) putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) - stack_root = "/usr/hdp" - if cluster_env and "stack_root" in cluster_env: - stack_root = cluster_env["stack_root"] + stack_root = self.getStackRoot(services) timeline_plugin_classes_values = [] timeline_plugin_classpath_values = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 8e08d82..67f7fe0 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -25,6 +25,7 @@ import re import socket import string import traceback +import json import sys import logging from math import ceil, floor @@ -34,7 +35,6 @@ from urlparse import urlparse from resource_management.libraries.functions.data_structure_utils import get_from_dict from resource_management.core.exceptions import Fail - class StackAdvisor(object): """ Abstract class implemented by all stack advisors. Stack advisors advise on stack specific questions. @@ -2006,6 +2006,23 @@ class DefaultStackAdvisor(StackAdvisor): return mount_points + def getStackRoot(self, services): + """ + Gets the stack root associated with the stack + :param services: the services structure containing the current configurations + :return: the stack root as specified in the config or /usr/hdp + """ + cluster_env = self.getServicesSiteProperties(services, "cluster-env") + stack_root = "/usr/hdp" + if cluster_env and "stack_root" in cluster_env: + stack_root_as_str = cluster_env["stack_root"] + stack_roots = json.loads(stack_root_as_str) + stack_name = cluster_env["stack_name"] + if stack_name in stack_roots: + stack_root = stack_roots[stack_name] + + return stack_root + def isSecurityEnabled(self, services): """ Determines if security is enabled by testing the value of cluster-env/security enabled. http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 50f527d..bf0cbec 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -825,70 +825,80 @@ class TestHDP25StackAdvisor(TestCase): services = { "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - }, - { - "StackServiceComponents": { - "advertise_version": "true", - "cardinality": "1+", - "component_category": "SLAVE", - "component_name": "NODEMANAGER", - "display_name": "NodeManager", - "is_client": "false", - "is_master": "false", - "hostnames": [ - "c6403.ambari.apache.org" - ] - }, - "dependencies": [] - }, - ] - } + "StackServices": { + "service_name": "TEZ" + } + }, + { + "StackServices": { + "service_name": "SPARK" + } + }, + { + "StackServices": { + "service_name": "YARN", + }, + "Versions": { + "stack_version": "2.5" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "NODEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + } + } + ] + }, { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", + "StackServices": { + "service_name": "HIVE", + "service_version": "1.2.1.2.5", + "stack_name": "HDP", + "stack_version": "2.5" + }, + "components": [ + { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", + "StackServiceComponents": { + "advertise_version": "true", + "bulk_commands_display_name": "", + "bulk_commands_master_component_name": "", + "cardinality": "0-1", + "component_category": "MASTER", + "component_name": "HIVE_SERVER_INTERACTIVE", + "custom_commands": ["RESTART_LLAP"], + "decommission_allowed": "false", + "display_name": "HiveServer2 Interactive", + "has_bulk_commands_definition": "false", + "is_client": "false", + "is_master": "true", + "reassign_allowed": "false", + "recovery_enabled": "false", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.5", + "hostnames": ["c6401.ambari.apache.org"] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + ] + } ], "changed-configurations": [ { @@ -898,6 +908,12 @@ class TestHDP25StackAdvisor(TestCase): } ], "configurations": { + "cluster-env": { + "properties": { + "stack_root": "{\"HDP\":\"/usr/hdp\"}", + "stack_name": "HDP" + }, + }, "capacity-scheduler": { "properties": { "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n' @@ -960,7 +976,8 @@ class TestHDP25StackAdvisor(TestCase): "tez.am.resource.memory.mb": "341" } } - } + }, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} } clusterData = { @@ -990,6 +1007,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default') self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'], + 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*') self.assertTrue('hive-interactive-env' not in configurations) self.assertTrue('property_attributes' not in configurations)