Repository: ambari Updated Branches: refs/heads/branch-2.4 80371f6ec -> 33adc47a4
AMBARI-18046. Falcon Service check failed after deleting and adding back falcon (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/33adc47a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/33adc47a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/33adc47a Branch: refs/heads/branch-2.4 Commit: 33adc47a426b42426532c00a610bbdc52228c421 Parents: 80371f6 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Aug 5 17:16:58 2016 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Aug 5 17:16:58 2016 +0300 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 2 + .../libraries/functions/package_conditions.py | 19 ++++- .../libraries/functions/setup_atlas_hook.py | 11 ++- .../server/upgrade/UpgradeCatalog240.java | 18 ++++ .../0.5.0.2.1/configuration/falcon-env.xml | 2 + .../configuration/falcon-startup.properties.xml | 2 +- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 14 +++- .../0.5.0.2.1/package/scripts/params_linux.py | 20 +++++ .../HDP/2.0.6/properties/stack_features.json | 11 +++ .../configuration/falcon-startup.properties.xml | 2 +- .../configuration/falcon-startup.properties.xml | 2 +- .../stacks/HDP/2.2/services/FALCON/metainfo.xml | 8 ++ .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 2 +- .../stacks/HDP/2.3/services/stack_advisor.py | 36 -------- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 2 +- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 2 +- .../configuration/falcon-startup.properties.xml | 2 +- .../stacks/HDP/2.5/services/FALCON/metainfo.xml | 28 +++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 8 -- .../configuration/falcon-startup.properties.xml | 2 +- .../server/upgrade/UpgradeCatalog240Test.java | 37 ++++++++- .../stacks/2.3/common/test_stack_advisor.py | 87 -------------------- 22 files changed, 172 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 0d6f2b2..c5cbb3f 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -91,6 +91,8 @@ class StackFeature: ATLAS_RANGER_PLUGIN_SUPPORT = "atlas_ranger_plugin_support" ATLAS_UPGRADE_SUPPORT = "atlas_upgrade_support" ATLAS_CONF_DIR_IN_PATH = "atlas_conf_dir_in_path" + FALCON_ATLAS_SUPPORT_2_3 = "falcon_atlas_support_2_3" + FALCON_ATLAS_SUPPORT = "falcon_atlas_support" RANGER_PID_SUPPORT = "ranger_pid_support" RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support" RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change" http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 472b8d9..a8b5827 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -24,9 +24,12 @@ __all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_colle import os from resource_management.libraries.script import Script +from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version -def _has_applicable_local_component(config, components): +def _has_local_components(config, components, indicator_function = any): if 'role' not in config: return False if config['role'] == 'install_packages': @@ -34,10 +37,13 @@ def _has_applicable_local_component(config, components): # Check if if 'localComponents' not in config: return False - return any([component in config['localComponents'] for component in components]) + return indicator_function([component in config['localComponents'] for component in components]) else: return config['role'] in components +def _has_applicable_local_component(config, components): + return _has_local_components(config, components, any) + def should_install_lzo(): config = Script.get_config() io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) @@ -88,6 +94,15 @@ def should_install_hive_atlas(): has_atlas = len(atlas_hosts) > 0 return has_atlas +def should_install_falcon_atlas_hook(): + config = Script.get_config() + stack_version_unformatted = config['hostLevelParams']['stack_version'] + stack_version_formatted = format_stack_version(stack_version_unformatted) + if check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, stack_version_formatted) \ + or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted): + return _has_local_components(config, ['FALCON_SERVER', 'ATLAS_SERVER'], all) + return False + def should_install_ranger_tagsync(): config = Script.get_config() ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", []) http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py index acdb1b9..26a0dd6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py @@ -28,7 +28,9 @@ from resource_management.libraries.resources.properties_file import PropertiesFi from resource_management.libraries.functions.format import format from resource_management.libraries.functions.default import default from resource_management.core.resources.system import Link +from resource_management.core.resources.packaging import Package from resource_management.core.logger import Logger +from ambari_commons import OSCheck from ambari_commons.constants import SERVICE ''' @@ -169,4 +171,11 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir): atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name) source_lib_file_name = os.path.join(jar_source_dir, file_name) if os.path.isfile(atlas_hook_file_name): - Link(source_lib_file_name, to=atlas_hook_file_name) \ No newline at end of file + Link(source_lib_file_name, to=atlas_hook_file_name) + +def install_atlas_hook_packages(): + import params + + if not params.host_sys_prepped: + Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index 8725050..5c96e5a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -2206,6 +2206,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { /** * Updates the Falcon-related configurations for the clusters managed by this Ambari * Removes falcon_store_uri from falcon-env. + * Appends '{{atlas_application_class_addition}}' to *.application.services from falcon-startup.properties if it doesn't contain it. * * @throws AmbariException if an error occurs while updating the configurations */ @@ -2215,6 +2216,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); for (final Cluster cluster : clusterMap.values()) { + // Remove falcon_store_uri from falcon-env. Config falconEnvConfig = cluster.getDesiredConfigByType("falcon-env"); if (falconEnvConfig != null) { Map<String, String> falconEnvEnvProperties = falconEnvConfig.getProperties(); @@ -2223,6 +2225,22 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { removeConfigurationPropertiesFromCluster(cluster, "falcon-env", Collections.singleton("falcon_store_uri")); } } + + // Update falcon-startup.properties/*.application.services. + // Ensure {{atlas_application_class_addition}} + Config falconStartupConfig = cluster.getDesiredConfigByType("falcon-startup.properties"); + if (falconStartupConfig != null) { + final String applicationServicesPropertyName = "*.application.services"; + String value = falconStartupConfig.getProperties().get(applicationServicesPropertyName); + if (value != null) { + final String atlasApplicationClassAddition = "{{atlas_application_class_addition}}"; + if (!(value.contains(atlasApplicationClassAddition) || value.contains("org.apache.falcon.atlas.service.AtlasService") || value.contains("org.apache.atlas.falcon.service.AtlasService"))) { + LOG.info("Appending '{}' to *.application.services from falcon-startup.properties since it doesn't contain it", atlasApplicationClassAddition); + String newValue = value.trim().concat(atlasApplicationClassAddition); + updateConfigurationPropertiesForCluster(cluster, "falcon-startup.properties", Collections.singletonMap(applicationServicesPropertyName, newValue), null, true, false); + } + } + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml index 34cf595..48984fd 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml @@ -163,7 +163,9 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}} #export FALCON_EXPANDED_WEBAPP_DIR= # Add the Atlas Falcon hook to the Falcon classpath +{% if falcon_atlas_support %} export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH} +{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml index f8681db..c0d710d 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml @@ -70,7 +70,7 @@ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService + org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}} </value> <description/> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 5712516..f214b49 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -34,8 +34,10 @@ from resource_management.libraries.script import Script from resource_management.libraries.resources import PropertiesFile from resource_management.libraries.functions import format from resource_management.libraries.functions.show_logs import show_logs -from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, install_atlas_hook_packages, setup_atlas_jar_symlinks +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import get_user_call_output +from resource_management.libraries.functions import StackFeature from ambari_commons.constants import SERVICE from resource_management.core.logger import Logger @@ -115,10 +117,18 @@ def falcon(type, action = None, upgrade_type=None): cd_access = "a") # Generate atlas-application.properties.xml file - if has_atlas_in_cluster(): + if params.falcon_atlas_support: + # If Atlas is added later than Falcon, this package will be absent. + install_atlas_hook_packages() + atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) + # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH. + # Setup symlinks for older versions. + if params.current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, params.current_version_formatted): + setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib) + if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index c302eeb..f59d77c 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -151,6 +151,8 @@ for host in atlas_hosts: has_atlas_server_on_host = True break +falcon_atlas_support = False + # Path to add to environment variable atlas_hook_cp = "" if has_atlas_in_cluster(): @@ -158,12 +160,30 @@ if has_atlas_in_cluster(): # Only append /etc/atlas/conf to classpath if on HDP 2.4.* and atlas server is running on this host. if has_atlas_server_on_host: + # stack_version doesn't contain a minor number of the stack (only first two numbers: 2.3). Get it from current_version_formatted + falcon_atlas_support = current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, current_version_formatted) \ + or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted) + if check_stack_feature(StackFeature.ATLAS_CONF_DIR_IN_PATH, stack_version_formatted): atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else format('{stack_root}/current/atlas-server/conf') atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep elif check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, stack_version_formatted): atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*') + +atlas_application_class_addition = "" +if falcon_atlas_support: + # Some stack versions do not support Atlas Falcon hook. See stack_features.json + # Packaging was different in older versions. + if current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, current_version_formatted): + atlas_application_class_addition = ",\\\norg.apache.falcon.atlas.service.AtlasService" + atlas_plugin_package = "atlas-metadata*-falcon-plugin" + atlas_ubuntu_plugin_package = "atlas-metadata.*-falcon-plugin" + else: + atlas_application_class_addition = ",\\\norg.apache.atlas.falcon.service.AtlasService" + atlas_plugin_package = "atlas-metadata*-hive-plugin" + atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" + #endregion hdfs_site = config['configurations']['hdfs-site'] http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index da577ac..ed995b2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -232,6 +232,17 @@ "min_version": "2.5.0.0" }, { + "name": "falcon_atlas_support_2_3", + "description": "Falcon Atlas integration support for 2.3 stack", + "min_version": "2.3.99.0", + "max_version": "2.4.0.0" + }, + { + "name": "falcon_atlas_support", + "description": "Falcon Atlas integration", + "min_version": "2.5.0.0" + }, + { "name": "hbase_home_directory", "description": "Hbase home directory in HDFS needed for HBASE backup", "min_version": "2.5.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml index 5ae2a63..6b954e7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml @@ -70,7 +70,7 @@ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService + org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}} </value> <description/> <on-ambari-upgrade add="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml index 694d960..19f8b50 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml @@ -35,7 +35,7 @@ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ org.apache.falcon.service.LogCleanupService,\ - org.apache.falcon.metadata.MetadataMappingService + org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}} </value> <description>Falcon Services</description> <on-ambari-upgrade add="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml index 6b6bfd9..0b860cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml @@ -29,6 +29,10 @@ <package> <name>falcon_${stack_version}</name> </package> + <package> + <name>atlas-metadata_${stack_version}-falcon-plugin</name> + <condition>should_install_falcon_atlas_hook</condition> + </package> </packages> </osSpecific> <osSpecific> @@ -37,6 +41,10 @@ <package> <name>falcon-${stack_version}</name> </package> + <package> + <name>atlas-metadata-${stack_version}-falcon-plugin</name> + <condition>should_install_falcon_atlas_hook</condition> + </package> </packages> </osSpecific> </osSpecifics> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index 402e349..5f420f9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -234,7 +234,7 @@ <definition xsi:type="configure" id="hdp_2_2_0_0_falcon_application_services"> <type>falcon-startup.properties</type> <set key="*.application.services" - value="org.apache.falcon.security.AuthenticationInitializationService,\
 org.apache.falcon.workflow.WorkflowJobEndNotificationService,\
 org.apache.falcon.service.ProcessSubscriberService,\
 org.apache.falcon.entity.store.ConfigurationStore,\
 org.apache.falcon.rerun.service.RetryService,\
 org.apache.falcon.rerun.service.LateRunService,\
 org.apache.falcon.service.LogCleanupService,\
 org.apache.falcon.metadata.MetadataMappingService"/> + value="org.apache.falcon.security.AuthenticationInitializationService,\
 org.apache.falcon.workflow.WorkflowJobEndNotificationService,\
 org.apache.falcon.service.ProcessSubscriberService,\
 org.apache.falcon.entity.store.ConfigurationStore,\
 org.apache.falcon.rerun.service.RetryService,\
 org.apache.falcon.rerun.service.LateRunService,\
 org.apache.falcon.service.LogCleanupService,\
 org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 9f2caf9..9d75cee 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -26,7 +26,6 @@ import socket # Local Imports from resource_management.core.logger import Logger -from resource_management.libraries.functions.version import compare_versions DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"} @@ -81,7 +80,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "KAFKA": self.recommendKAFKAConfigurations, "RANGER": self.recommendRangerConfigurations, "RANGER_KMS": self.recommendRangerKMSConfigurations, - "FALCON": self.recommendFalconConfigurations, "STORM": self.recommendStormConfigurations, "SQOOP": self.recommendSqoopConfigurations } @@ -737,40 +735,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): putStormStartupProperty(notifier_plugin_property, notifier_plugin_value) - def recommendFalconConfigurations(self, configurations, clusterData, services, hosts): - # In 2.5, the classname changed. The 2.5 stack advisor method will - # call this with attribute set. 2.3 will use the old method name - if hasattr(self, "atlasFalconHookClassName"): - atlas_application_class = self.atlasFalconHookClassName - else: - atlas_application_class = "org.apache.falcon.atlas.service.AtlasService" - putFalconStartupProperty = self.putProperty(configurations, "falcon-startup.properties", services) - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - - # atlas - application_services_property = "*.application.services" - if "falcon-startup.properties" in services["configurations"] and application_services_property in services["configurations"]["falcon-startup.properties"]["properties"]: - application_services_value = services["configurations"]["falcon-startup.properties"]["properties"][application_services_property] - else: - application_services_value = " " - - include_atlas = "ATLAS" in servicesList - if include_atlas and atlas_application_class not in application_services_value: - if application_services_value == " ": - application_services_value = atlas_application_class - else: - application_services_value = application_services_value + "," + atlas_application_class - if not include_atlas and atlas_application_class in application_services_value: - application_classes = [] - for application_class in application_services_value.split(","): - if application_class != atlas_application_class and application_class != " ": - application_classes.append(application_class) - if application_classes: - application_services_value = ",".join(application_classes) - else: - application_services_value = " " - putFalconStartupProperty(application_services_property, application_services_value) - def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 7661205..9b4ef8c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -526,7 +526,7 @@ <changes> <definition xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"> <type>falcon-startup.properties</type> - <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService"/> + <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index d230a68..29134a3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -345,7 +345,7 @@ <changes> <definition xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"> <type>falcon-startup.properties</type> - <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService"/> + <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml index b0779bf..0508638 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml @@ -37,7 +37,7 @@ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ org.apache.falcon.service.LogCleanupService,\ - org.apache.falcon.metadata.MetadataMappingService + org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}} </value> <description>Falcon application services</description> <on-ambari-upgrade add="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml index 6586937..79296c3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml @@ -21,6 +21,34 @@ <service> <name>FALCON</name> <version>0.10.0.2.5</version> + + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>falcon_${stack_version}</name> + </package> + <package> + <name>atlas-metadata_${stack_version}-hive-plugin</name> + <condition>should_install_falcon_atlas_hook</condition> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>falcon-${stack_version}</name> + </package> + <package> + <name>atlas-metadata-${stack_version}-hive-plugin</name> + <condition>should_install_falcon_atlas_hook</condition> + </package> + </packages> + </osSpecific> + </osSpecifics> + <configuration-dependencies> <config-type>application-properties</config-type> </configuration-dependencies> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index d94cb09..68b03ad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -79,13 +79,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor): services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"}) putOozieEnvProperty("oozie_admin_users", newAdminUsers) - - def recommendFalconConfigurations(self, configurations, clusterData, services, hosts): - # Set the classname for the Falcon hook to 2.5 classname = all other 2.3 - # stack advisor changes are needed. - self.atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService" - super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts) - def createComponentLayoutRecommendations(self, services, hosts): parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations( services, hosts) @@ -396,7 +389,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor): "ATLAS": self.recommendAtlasConfigurations, "RANGER_KMS": self.recommendRangerKMSConfigurations, "STORM": self.recommendStormConfigurations, - "FALCON": self.recommendFalconConfigurations, "OOZIE": self.recommendOozieConfigurations, "SPARK2": self.recommendSpark2Configurations } http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml index adc3691..c7d4304 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml @@ -34,7 +34,7 @@ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService + org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}} </value> <description>Falcon Services</description> <on-ambari-upgrade add="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index e125165..f7c33ad 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -784,6 +784,31 @@ public class UpgradeCatalog240Test { propertiesExpectedFalconEnv.put("falcon_store_uri", "file:///hadoop/falcon/store"); propertiesExpectedFalconEnv.put("property", "value"); + final String applicationServicesOldPropertyValue = + "org.apache.falcon.security.AuthenticationInitializationService,\\\n" + + " org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n" + + " org.apache.falcon.service.ProcessSubscriberService,\\\n" + + " org.apache.falcon.entity.store.ConfigurationStore,\\\n" + + " org.apache.falcon.rerun.service.RetryService,\\\n" + + " org.apache.falcon.rerun.service.LateRunService,\\\n" + + " org.apache.falcon.service.LogCleanupService,\\\n" + + " org.apache.falcon.metadata.MetadataMappingService"; + + final String applicationServicesExpectedPropertyValue = + "org.apache.falcon.security.AuthenticationInitializationService,\\\n" + + " org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n" + + " org.apache.falcon.service.ProcessSubscriberService,\\\n" + + " org.apache.falcon.entity.store.ConfigurationStore,\\\n" + + " org.apache.falcon.rerun.service.RetryService,\\\n" + + " org.apache.falcon.rerun.service.LateRunService,\\\n" + + " org.apache.falcon.service.LogCleanupService,\\\n" + + " org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"; + + final Config falconStartupConfig = easyMockSupport.createNiceMock(Config.class); + + final Map<String, String> falconStartupConfigProperties= new HashMap<String, String>(); + falconStartupConfigProperties.put("*.application.services", applicationServicesOldPropertyValue); + falconStartupConfigProperties.put("property", "value"); final Injector mockInjector = Guice.createInjector(new Module() { @Override public void configure(Binder binder) { @@ -805,9 +830,16 @@ public class UpgradeCatalog240Test { expect(mockClusterExpected.getDesiredConfigByType("falcon-env")).andReturn(mockFalconEnv).atLeastOnce(); expect(mockFalconEnv.getProperties()).andReturn(propertiesExpectedFalconEnv).anyTimes(); + expect(mockClusterExpected.getDesiredConfigByType("falcon-startup.properties")).andReturn(falconStartupConfig).atLeastOnce(); + expect(falconStartupConfig.getProperties()).andReturn(falconStartupConfigProperties).anyTimes(); + Capture<Map<String, String>> falconCapture = newCapture(); expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"), - capture(falconCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); + capture(falconCapture), anyString(), (Map<String, Map<String, String>>) anyObject())).andReturn(null).once(); + + Capture<Map<String, String>> falconStartupCapture = newCapture(); + expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-startup.properties"), + capture(falconStartupCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); easyMockSupport.replayAll(); mockInjector.getInstance(UpgradeCatalog240.class).updateFalconConfigs(); @@ -815,6 +847,9 @@ public class UpgradeCatalog240Test { assertEquals("value", falconCapture.getValue().get("property")); assertNull(falconCapture.getValue().get("falcon_store_uri")); + + assertEquals("value", falconStartupCapture.getValue().get("property")); + assertEquals(applicationServicesExpectedPropertyValue, falconStartupCapture.getValue().get("*.application.services")); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 1af87e4..272269d 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1782,93 +1782,6 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - - def test_recommendFalconConfigurations(self): - self.maxDiff = None - configurations = { - "falcon-startup.properties": { - "properties": { - "*.application.services": "foo" - } - } - } - clusterData = { - "cpu": 4, - "mapMemory": 3000, - "amMemory": 2000, - "reduceMemory": 2056, - "containers": 3, - "ramPerContainer": 256 - } - expected = { - 'falcon-startup.properties': { - 'properties': { - '*.application.services': 'foo,org.apache.falcon.atlas.service.AtlasService', - } - } - } - services = { - "services": [ - { - "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS", - "StackServices": { - "service_name": "ATLAS", - "service_version": "2.6.0.2.2", - "stack_name": "HDP", - "stack_version": "2.3" - }, - "components": [ - { - "StackServiceComponents": { - "advertise_version": "false", - "cardinality": "1", - "component_category": "MASTER", - "component_name": "ATLAS_SERVER", - "display_name": "Atlas Server", - "is_client": "false", - "is_master": "true", - "hostnames": [] - }, - "dependencies": [] - } - ] - }, - ], - "configurations": { - "falcon-startup.properties": { - "properties": { - "*.application.services": "foo" - } - } - }, - "changed-configurations": [ ] - - } - hosts = { - "items" : [ - { - "href" : "/api/v1/hosts/c6401.ambari.apache.org", - "Hosts" : { - "cpu_count" : 1, - "host_name" : "c6401.ambari.apache.org", - "os_arch" : "x86_64", - "os_type" : "centos6", - "ph_cpu_count" : 1, - "public_host_name" : "c6401.ambari.apache.org", - "rack_info" : "/default-rack", - "total_mem" : 1922680 - } - } - ] - } - - self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} - self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - def test_recommendSqoopConfigurations(self): self.maxDiff = None configurations = {
