AMBARI-18062. Review the implementation of the Falcon/Atlas hook enabling/disabling (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03536e16 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03536e16 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03536e16 Branch: refs/heads/branch-2.4 Commit: 03536e1699f15fa654d4104972b27fc8072c9e63 Parents: 8444050 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Mon Aug 8 18:36:39 2016 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Mon Aug 8 18:36:39 2016 +0300 ---------------------------------------------------------------------- .../libraries/functions/setup_atlas_hook.py | 11 +++++------ .../ambari/server/upgrade/UpgradeCatalog240.java | 19 ++++++++++++++++++- .../0.5.0.2.1/configuration/falcon-env.xml | 2 +- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 3 ++- .../0.5.0.2.1/package/scripts/params_linux.py | 2 +- .../server/upgrade/UpgradeCatalog240Test.java | 15 +++++++++++++++ 6 files changed, 42 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py index 26a0dd6..2fdcf9d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py @@ -173,9 +173,8 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir): if os.path.isfile(atlas_hook_file_name): Link(source_lib_file_name, to=atlas_hook_file_name) -def install_atlas_hook_packages(): - import params - - if not params.host_sys_prepped: - Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) \ No newline at end of file +def install_atlas_hook_packages(atlas_plugin_package, atlas_ubuntu_plugin_package, host_sys_prepped, + agent_stack_retry_on_unavailability, agent_stack_retry_count): + if not host_sys_prepped: + Package(atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else atlas_plugin_package, + retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index 8c22eeb..b3613af 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -2201,6 +2201,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { /** * Updates the Falcon-related configurations for the clusters managed by this Ambari * Removes falcon_store_uri from falcon-env. + * Appends Atlas hook classpath to FALCON_EXTRA_CLASS_PATH from falcon-env if it doesn't contain it * Appends '{{atlas_application_class_addition}}' to *.application.services from falcon-startup.properties if it doesn't contain it. * * @throws AmbariException if an error occurs while updating the configurations @@ -2211,14 +2212,30 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); for (final Cluster cluster : clusterMap.values()) { - // Remove falcon_store_uri from falcon-env. Config falconEnvConfig = cluster.getDesiredConfigByType("falcon-env"); if (falconEnvConfig != null) { + // Remove falcon_store_uri from falcon-env. Map<String, String> falconEnvEnvProperties = falconEnvConfig.getProperties(); if (falconEnvEnvProperties.containsKey("falcon_store_uri")) { LOG.info("Removing property falcon_store_uri from falcon-env"); removeConfigurationPropertiesFromCluster(cluster, "falcon-env", Collections.singleton("falcon_store_uri")); } + + // Append Atlas hook classpath to FALCON_EXTRA_CLASS_PATH from falcon-env if it doesn't contain it + // This is necessary for stack upgrades to 2.5 after the Ambari upgrade + final String envContentPropertyName = "content"; + String contentValue = falconEnvConfig.getProperties().get(envContentPropertyName); + if (contentValue != null) { + final String atlasHookCPAddition = "{{atlas_hook_cp}}"; + if (!contentValue.contains(atlasHookCPAddition)) { + LOG.info("Appending '{}' to FALCON_EXTRA_CLASS_PATH from falcon-env since it doesn't contain it", atlasHookCPAddition); + String newValue = contentValue + "\n\n{% if falcon_atlas_support %}\n" + + "# Add the Atlas Falcon hook to the Falcon classpath\n" + + "export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}\n" + + "{% endif %}"; + updateConfigurationPropertiesForCluster(cluster, "falcon-env", Collections.singletonMap(envContentPropertyName, newValue), null, true, false); + } + } } // Update falcon-startup.properties/*.application.services. http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml index 48984fd..09cced6 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml @@ -162,8 +162,8 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. #export FALCON_EXPANDED_WEBAPP_DIR= -# Add the Atlas Falcon hook to the Falcon classpath {% if falcon_atlas_support %} +# Add the Atlas Falcon hook to the Falcon classpath export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH} {% endif %} </value> http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 8025e56..7a5cecb 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -120,7 +120,8 @@ def falcon(type, action = None, upgrade_type=None): # Generate atlas-application.properties.xml file if params.falcon_atlas_support: # If Atlas is added later than Falcon, this package will be absent. - install_atlas_hook_packages() + install_atlas_hook_packages(params.atlas_plugin_package, params.atlas_ubuntu_plugin_package, params.host_sys_prepped, + params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count) atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 2220480..a3a4e54 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -166,7 +166,7 @@ if has_atlas_in_cluster(): atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep elif check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, stack_version_formatted): - atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*') + atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*') + os.pathsep atlas_application_class_addition = "" if falcon_atlas_support: http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index 3b42984..734cc7c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -779,6 +779,7 @@ public class UpgradeCatalog240Test { final Map<String, String> propertiesExpectedFalconEnv = new HashMap<String, String>(); propertiesExpectedFalconEnv.put("falcon_store_uri", "file:///hadoop/falcon/store"); + propertiesExpectedFalconEnv.put("content", "# content"); propertiesExpectedFalconEnv.put("property", "value"); final String applicationServicesOldPropertyValue = @@ -834,6 +835,10 @@ public class UpgradeCatalog240Test { expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"), capture(falconCapture), anyString(), (Map<String, Map<String, String>>) anyObject())).andReturn(null).once(); + Capture<Map<String, String>> falconCapture2 = newCapture(); + expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"), + capture(falconCapture2), anyString(), (Map<String, Map<String, String>>) anyObject())).andReturn(null).once(); + Capture<Map<String, String>> falconStartupCapture = newCapture(); expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-startup.properties"), capture(falconStartupCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); @@ -842,9 +847,19 @@ public class UpgradeCatalog240Test { mockInjector.getInstance(UpgradeCatalog240.class).updateFalconConfigs(); easyMockSupport.verifyAll(); + final String expectredEnvContent = "# content\n" + + "\n" + + "{% if falcon_atlas_support %}\n" + + "# Add the Atlas Falcon hook to the Falcon classpath\n" + + "export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}\n" + + "{% endif %}"; + assertEquals("value", falconCapture.getValue().get("property")); + assertEquals("# content", falconCapture.getValue().get("content")); assertNull(falconCapture.getValue().get("falcon_store_uri")); + assertEquals(expectredEnvContent, falconCapture2.getValue().get("content")); + assertEquals("value", falconStartupCapture.getValue().get("property")); assertEquals(applicationServicesExpectedPropertyValue, falconStartupCapture.getValue().get("*.application.services")); }
