AMBARI-19018 - Services should be able to specify their own resources subdirs for syncing to agents
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/232a9d87 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/232a9d87 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/232a9d87 Branch: refs/heads/branch-dev-patch-upgrade Commit: 232a9d87e72dfb87e0d25f6a4cba6738e27beb53 Parents: 60a6bd4 Author: Tim Thorpe <ttho...@apache.org> Authored: Fri Dec 2 08:15:09 2016 -0800 Committer: Tim Thorpe <ttho...@apache.org> Committed: Fri Dec 2 08:15:09 2016 -0800 ---------------------------------------------------------------------- .../ambari_agent/CustomServiceOrchestrator.py | 7 +- .../src/main/python/ambari_agent/FileCache.py | 12 ++- .../TestCustomServiceOrchestrator.py | 30 +----- .../test/python/ambari_agent/TestFileCache.py | 10 +- .../ambari/server/agent/ExecutionCommand.java | 1 + .../AmbariManagementControllerImpl.java | 9 ++ .../ambari/server/state/ComponentInfo.java | 14 +++ .../python/ambari_server/serverConfiguration.py | 14 +-- .../AMBARI_METRICS/0.1.0/metainfo.xml | 1 + .../AmbariManagementControllerTest.java | 9 ++ .../server/orm/InMemoryDefaultTestModule.java | 5 + ambari-server/src/test/python/TestMpacks.py | 102 +++++++++++++------ .../src/test/resources/dashboards/README.txt | 18 ++++ .../stacks/HDP/0.1/services/HDFS/metainfo.xml | 1 + 14 files changed, 150 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 7d61611..11c8cbe 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -154,12 +154,7 @@ class CustomServiceOrchestrator(): self.file_cache.get_host_scripts_base_dir(server_url_prefix) hook_dir = self.file_cache.get_hook_base_dir(command, server_url_prefix) base_dir = self.file_cache.get_service_base_dir(command, server_url_prefix) - from ActionQueue import ActionQueue # To avoid cyclic dependency - if self.COMMAND_TYPE in command and command[self.COMMAND_TYPE] == ActionQueue.EXECUTION_COMMAND: - logger.info("Found it - " + str(command[self.COMMAND_TYPE]) + " yeah") - # limiting to only EXECUTION_COMMANDs for now - # TODO need a design for limiting to specific role/component such as METRICS_GRAFANA - self.file_cache.get_dashboard_base_dir(server_url_prefix) + self.file_cache.get_custom_resources_subdir(command, server_url_prefix) script_path = self.resolve_script_path(base_dir, script) script_tuple = (script_path, base_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/main/python/ambari_agent/FileCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/FileCache.py b/ambari-agent/src/main/python/ambari_agent/FileCache.py index 83ac373..a9ea8f6 100644 --- a/ambari-agent/src/main/python/ambari_agent/FileCache.py +++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py @@ -45,7 +45,6 @@ class FileCache(): STACKS_CACHE_DIRECTORY="stacks" COMMON_SERVICES_DIRECTORY="common-services" CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions" - DASHBOARD_DIRECTORY="dashboards" HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts" HASH_SUM_FILE=".hash" ARCHIVE_NAME="archive.zip" @@ -100,12 +99,17 @@ class FileCache(): server_url_prefix) - def get_dashboard_base_dir(self, server_url_prefix): + def get_custom_resources_subdir(self, command, server_url_prefix): """ - Returns a base directory for dashboards + Returns a custom directory which must be a subdirectory of the resources dir """ + try: + custom_dir = command['commandParams']['custom_folder'] + except KeyError: + return None + return self.provide_directory(self.cache_dir, - self.DASHBOARD_DIRECTORY, + custom_dir, server_url_prefix) http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py index 0304adc..5323d9a 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py @@ -220,7 +220,7 @@ class TestCustomServiceOrchestrator(TestCase): except AgentException: pass # Expected - @patch.object(FileCache, "get_dashboard_base_dir") + @patch.object(FileCache, "get_custom_resources_subdir") @patch.object(CustomServiceOrchestrator, "resolve_script_path") @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") @patch.object(FileCache, "get_host_scripts_base_dir") @@ -235,7 +235,7 @@ class TestCustomServiceOrchestrator(TestCase): get_host_scripts_base_dir_mock, resolve_hook_script_path_mock, resolve_script_path_mock, - get_dashboard_base_dir_mock): + get_custom_resources_subdir_mock): FileCache_mock.return_value = None command = { @@ -267,7 +267,6 @@ class TestCustomServiceOrchestrator(TestCase): unix_process_id = 111 orchestrator.commands_in_progress = {command['taskId']: unix_process_id} get_hook_base_dir_mock.return_value = "/hooks/" - get_dashboard_base_dir_mock.return_value = "/dashboards/" # normal run case run_file_mock.return_value = { 'stdout' : 'sss', @@ -278,7 +277,6 @@ class TestCustomServiceOrchestrator(TestCase): self.assertEqual(ret['exitcode'], 0) self.assertTrue(run_file_mock.called) self.assertEqual(run_file_mock.call_count, 3) - self.assertTrue(get_dashboard_base_dir_mock.called) run_file_mock.reset_mock() @@ -301,25 +299,6 @@ class TestCustomServiceOrchestrator(TestCase): run_file_mock.reset_mock() - # For role=METRICS_GRAFANA, dashboards should be sync'd - command['role'] = 'METRICS_GRAFANA' - get_dashboard_base_dir_mock.reset_mock() - get_dashboard_base_dir_mock.return_value = "/dashboards/" - - run_file_mock.return_value = { - 'stdout' : 'sss', - 'stderr' : 'eee', - 'exitcode': 0, - } - ret = orchestrator.runCommand(command, "out.txt", "err.txt") - self.assertEqual(ret['exitcode'], 0) - self.assertTrue(run_file_mock.called) - self.assertEqual(run_file_mock.call_count, 3) - self.assertTrue(get_dashboard_base_dir_mock.called) - - command['role'] = 'REGION_SERVER' - run_file_mock.reset_mock() - # unknown script type case command['commandParams']['script_type'] = "SOME_TYPE" ret = orchestrator.runCommand(command, "out.txt", "err.txt") @@ -332,7 +311,6 @@ class TestCustomServiceOrchestrator(TestCase): pass - @patch.object(FileCache, "get_dashboard_base_dir") @patch("ambari_commons.shell.kill_process_with_children") @patch.object(CustomServiceOrchestrator, "resolve_script_path") @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") @@ -347,8 +325,7 @@ class TestCustomServiceOrchestrator(TestCase): get_hook_base_dir_mock, get_service_base_dir_mock, get_host_scripts_base_dir_mock, resolve_hook_script_path_mock, resolve_script_path_mock, - kill_process_with_children_mock, - get_dashboard_base_dir_mock): + kill_process_with_children_mock): FileCache_mock.return_value = None command = { 'role' : 'REGION_SERVER', @@ -378,7 +355,6 @@ class TestCustomServiceOrchestrator(TestCase): unix_process_id = 111 orchestrator.commands_in_progress = {command['taskId']: unix_process_id} get_hook_base_dir_mock.return_value = "/hooks/" - get_dashboard_base_dir_mock.return_value = "/dashboards/" run_file_mock_return_value = { 'stdout' : 'killed', 'stderr' : 'killed', http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/test/python/ambari_agent/TestFileCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py index fbefc2b..00f6b69 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py +++ b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py @@ -119,10 +119,16 @@ class TestFileCache(TestCase): @patch.object(FileCache, "provide_directory") - def test_get_dashboard_base_dir(self, provide_directory_mock): + def test_get_custom_resources_subdir(self, provide_directory_mock): provide_directory_mock.return_value = "dummy value" fileCache = FileCache(self.config) - res = fileCache.get_dashboard_base_dir("server_url_pref") + command = { + 'commandParams': { + 'custom_folder' : 'dashboards' + } + } + + res = fileCache.get_custom_resources_subdir(command, "server_url_pref") self.assertEquals( pprint.pformat(provide_directory_mock.call_args_list[0][0]), "('/var/lib/ambari-agent/cache', 'dashboards', 'server_url_pref')") http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 29737ee..ef1ee4f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -343,6 +343,7 @@ public class ExecutionCommand extends AgentCommand { String SCRIPT_TYPE = "script_type"; String SERVICE_PACKAGE_FOLDER = "service_package_folder"; String HOOKS_FOLDER = "hooks_folder"; + String CUSTOM_FOLDER = "custom_folder"; String STACK_NAME = "stack_name"; String SERVICE_TYPE = "service_type"; String STACK_VERSION = "stack_version"; http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index ac9e405..0448b9f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -25,6 +25,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; @@ -2217,6 +2218,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle serviceInfo.getServicePackageFolder()); commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); + String customCacheDirectory = componentInfo.getCustomFolder(); + if (customCacheDirectory != null) { + File customCache = new File(configs.getResourceDirPath(), customCacheDirectory); + if (customCache.exists() && customCache.isDirectory()) { + commandParams.put(CUSTOM_FOLDER, customCacheDirectory); + } + } + String clusterName = cluster.getClusterName(); if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) { commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java index 1e494b4..bd9b798 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java @@ -133,6 +133,9 @@ public class ComponentInfo { private String timelineAppid; + @XmlElement(name="customFolder") + private String customFolder; + public ComponentInfo() { } @@ -158,6 +161,7 @@ public class ComponentInfo { clientConfigFiles = prototype.clientConfigFiles; timelineAppid = prototype.timelineAppid; reassignAllowed = prototype.reassignAllowed; + customFolder = prototype.customFolder; } public String getName() { @@ -396,6 +400,14 @@ public class ComponentInfo { this.reassignAllowed = reassignAllowed; } + public String getCustomFolder() { + return customFolder; + } + + public void setCustomFolder(String customFolder) { + this.customFolder = customFolder; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -426,6 +438,7 @@ public class ComponentInfo { if (name != null ? !name.equals(that.name) : that.name != null) return false; if (clientConfigFiles != null ? !clientConfigFiles.equals(that.clientConfigFiles) : that.clientConfigFiles != null) return false; + if (customFolder != null ? !customFolder.equals(that.customFolder) : that.customFolder != null) return false; return true; } @@ -450,6 +463,7 @@ public class ComponentInfo { result = 31 * result + (clientConfigFiles != null ? clientConfigFiles.hashCode() : 0); // NULL = 0, TRUE = 2, FALSE = 1 result = 31 * result + (versionAdvertisedField != null ? (versionAdvertisedField.booleanValue() ? 2 : 1) : 0); + result = 31 * result + (customFolder != null ? customFolder.hashCode() : 0); return result; } http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 9596e0d..6a6c89e 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -186,7 +186,6 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s DEFAULT_DB_NAME = "ambari" SECURITY_KEYS_DIR = "security.server.keys_dir" -DASHBOARD_PATH_PROPERTY = 'dashboards.path' EXTENSION_PATH_PROPERTY = 'extensions.path' COMMON_SERVICES_PATH_PROPERTY = 'common.services.path' MPACKS_STAGING_PATH_PROPERTY = 'mpacks.staging.path' @@ -398,8 +397,8 @@ class ServerConfigDefaults(object): self.EXTENSION_LOCATION_DEFAULT = "" self.COMMON_SERVICES_LOCATION_DEFAULT = "" self.MPACKS_STAGING_LOCATION_DEFAULT = "" - self.DASHBOARD_LOCATION_DEFAULT = "" self.SERVER_TMP_DIR_DEFAULT = "" + self.DASHBOARD_DIRNAME = "dashboards" self.DEFAULT_VIEWS_DIR = "" @@ -470,7 +469,6 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): self.EXTENSION_LOCATION_DEFAULT = "resources\\extensions" self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services" self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks" - self.DASHBOARD_LOCATION_DEFAULT = "resources\\dashboards" self.SERVER_TMP_DIR_DEFAULT = "data\\tmp" self.DEFAULT_VIEWS_DIR = "resources\\views" @@ -557,7 +555,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): self.EXTENSION_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/extensions") self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services") self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks") - self.DASHBOARD_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/dashboards") self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp") self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views") @@ -1448,13 +1445,8 @@ def get_mpacks_staging_location(properties): # Dashboard location # def get_dashboard_location(properties): - try: - dashboard_location = properties[DASHBOARD_PATH_PROPERTY] - except KeyError: - dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT - - if not dashboard_location: - dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT + resources_dir = get_resources_location(properties) + dashboard_location = os.path.join(resources_dir, configDefaults.DASHBOARD_DIRNAME) return dashboard_location # http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index 43a4320..740a91a 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -101,6 +101,7 @@ <category>MASTER</category> <cardinality>0-1</cardinality> <versionAdvertised>false</versionAdvertised> + <customFolder>dashboards</customFolder> <commandScript> <script>scripts/metrics_grafana.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 0fdaa46..db64f17 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1283,6 +1283,15 @@ public class AmbariManagementControllerTest { assertEquals("false", ec.getCommandParams().get("command_retry_enabled")); Map<String, Set<String>> chInfo = ec.getClusterHostInfo(); assertTrue(chInfo.containsKey("namenode_host")); + assertFalse(ec.getCommandParams().containsKey("custom_folder")); + + ec = controller.getExecutionCommand(cluster, + s1.getServiceComponent("DATANODE").getServiceComponentHost(host1), + RoleCommand.START); + assertEquals(cluster1, ec.getClusterName()); + assertNotNull(ec.getCommandParams()); + assertTrue(ec.getCommandParams().containsKey("custom_folder")); + assertEquals("dashboards", ec.getCommandParams().get("custom_folder")); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java index 6773176..a4a9e26 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java @@ -90,6 +90,7 @@ public class InMemoryDefaultTestModule extends AbstractModule { String stacks = "src/test/resources/stacks"; String version = "src/test/resources/version"; String sharedResourcesDir = "src/test/resources/"; + String resourcesDir = "src/test/resources/"; if (System.getProperty("os.name").contains("Windows")) { stacks = ClassLoader.getSystemClassLoader().getResource("stacks").getPath(); version = new File(new File(ClassLoader.getSystemClassLoader().getResource("").getPath()).getParent(), "version").getPath(); @@ -116,6 +117,10 @@ public class InMemoryDefaultTestModule extends AbstractModule { properties.setProperty(Configuration.SHARED_RESOURCES_DIR.getKey(), sharedResourcesDir); } + if (!properties.containsKey(Configuration.RESOURCES_DIR.getKey())) { + properties.setProperty(Configuration.RESOURCES_DIR.getKey(), resourcesDir); + } + try { install(Modules.override(new BeanDefinitionsCachingTestControllerModule(properties)).with(new AbstractModule() { @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/python/TestMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py index ba37856..32db005 100644 --- a/ambari-server/src/test/python/TestMpacks.py +++ b/ambari-server/src/test/python/TestMpacks.py @@ -64,7 +64,7 @@ def get_configs(): serverConfiguration.STACK_LOCATION_KEY : "/var/lib/ambari-server/resources/stacks", serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : "/var/lib/ambari-server/resources/common-services", serverConfiguration.EXTENSION_PATH_PROPERTY : "/var/lib/ambari-server/resources/extensions", - serverConfiguration.DASHBOARD_PATH_PROPERTY : "/var/lib/ambari-server/resources/dashboards", + serverConfiguration.RESOURCES_DIR_PROPERTY : "/var/lib/ambari-server/resources", serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory, serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp", serverConfiguration.JDBC_DATABASE_PROPERTY: "postgres" @@ -100,9 +100,11 @@ class TestMpacks(TestCase): fail = True self.assertTrue(fail) + @patch("os.path.exists") @patch("ambari_server.setupMpacks.get_YN_input") @patch("ambari_server.setupMpacks.run_mpack_install_checker") - def test_validate_purge(self, run_mpack_install_checker_mock, get_YN_input_mock): + @patch("ambari_server.setupMpacks.get_ambari_properties") + def test_validate_purge(self, get_ambari_properties_mock, run_mpack_install_checker_mock, get_YN_input_mock, os_path_exists_mock): options = self._create_empty_options_mock() options.purge = True purge_list = options.purge_list.split(',') @@ -112,6 +114,7 @@ class TestMpacks(TestCase): replay_mode = False run_mpack_install_checker_mock.return_value = (0, "No errors found", "") get_YN_input_mock.return_value = True + os_path_exists_mock.return_value = True fail = False try: @@ -160,22 +163,28 @@ class TestMpacks(TestCase): extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY] common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] - os_path_exists_mock.return_value = False + os_path_exists_mock.side_effect = [True] purge_stacks_and_mpacks(None) - os_path_exists_calls = [] + os_path_exists_calls = [ + call('/var/lib/ambari-server/resources'), + ] os_path_exists_mock.assert_has_calls(os_path_exists_calls) + os_path_exists_mock.side_effect = [True, False, False] purge_stacks_and_mpacks(options.purge_list.split(",")) os_path_exists_calls = [ + call('/var/lib/ambari-server/resources'), call(stacks_directory), call(mpacks_directory) ] os_path_exists_mock.assert_has_calls(os_path_exists_calls) options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) + os_path_exists_mock.side_effect = [True, False, False, False] purge_stacks_and_mpacks(options.purge_list.split(",")) os_path_exists_calls = [ + call('/var/lib/ambari-server/resources'), call(stacks_directory), call(common_services_directory), call(mpacks_directory) @@ -183,8 +192,10 @@ class TestMpacks(TestCase): os_path_exists_mock.assert_has_calls(os_path_exists_calls) options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) + os_path_exists_mock.side_effect = [True, False, False, False] purge_stacks_and_mpacks(options.purge_list.split(",")) os_path_exists_calls = [ + call('/var/lib/ambari-server/resources'), call(stacks_directory), call(extensions_directory), call(mpacks_directory) @@ -193,10 +204,13 @@ class TestMpacks(TestCase): options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) options.replay_mode = True + os_path_exists_mock.side_effect = [True, False, False, False] purge_stacks_and_mpacks(options.purge_list.split(",")) os_path_exists_calls = [ + call('/var/lib/ambari-server/resources'), call(stacks_directory), - call(common_services_directory) + call(common_services_directory), + call(mpacks_directory) ] os_path_exists_mock.assert_has_calls(os_path_exists_calls) @@ -266,10 +280,11 @@ class TestMpacks(TestCase): get_ambari_version_mock.return_value = "2.4.0.0" run_os_command_mock.return_value = (0, "", "") mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] - """ + os_path_exists_calls = [call('/tmp/mystack.tar.gz'), call('mpacks/mystack-ambari-mpack-1.0.0.0/mpack.json'), call('mpacks/mystack-ambari-mpack-1.0.0.0/hooks/before_install.py'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks'), call('/var/lib/ambari-server/resources/extensions'), call('/var/lib/ambari-server/resources/common-services'), @@ -277,12 +292,14 @@ class TestMpacks(TestCase): call(mpacks_directory + '/cache'), call('/var/lib/ambari-server/resources/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/common-services/SERVICEA'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/1.0/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/2.0/dashboards'), call('/var/lib/ambari-server/resources/common-services/SERVICEB'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/1.0.0/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/2.0.0/dashboards'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks/MYSTACK'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'), @@ -295,22 +312,26 @@ class TestMpacks(TestCase): call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEA/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEB/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/hooks/after_install.py')] - """ - os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False, False, - False, False, False, False, False, False, - False, False, False, False, False, False, False, False, - False, False, False, False, True] + + os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False, False, + False, True, False, False, False, False, False, False, True, False, + False, False, False, False, False, False, False, False, False, False, + True] get_ambari_properties_mock.return_value = configs shutil_move_mock.return_value = True - install_mpack(options) + try: + install_mpack(options) + except Exception as e: + print e stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY] common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY] mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] - dashboards_directory = serverConfiguration.get_dashboard_location(configs) mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.0") + resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY] + dashboards_directory = os.path.join(resources_directory, "dashboards") run_os_command_calls = [ call([ @@ -377,6 +398,7 @@ class TestMpacks(TestCase): "SERVICEB", None) ] + os_path_exists_mock.assert_has_calls(os_path_exists_calls) self.assertTrue(purge_stacks_and_mpacks_mock.called) run_os_command_mock.assert_has_calls(run_os_command_calls) os_mkdir_mock.assert_has_calls(os_mkdir_calls) @@ -403,8 +425,8 @@ class TestMpacks(TestCase): expand_mpack_mock.return_value = "mpacks/myextension-ambari-mpack-1.0.0.0" get_ambari_version_mock.return_value = "2.4.0.0" - os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False, - False, True, False, False, False] + os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False, + False, True, True, False, False, False] get_ambari_properties_mock.return_value = configs shutil_move_mock.return_value = True @@ -413,10 +435,12 @@ class TestMpacks(TestCase): extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY] mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] mpacks_staging_directory = os.path.join(mpacks_directory, "myextension-ambari-mpack-1.0.0.0") - dashboards_directory = serverConfiguration.get_dashboard_location(configs) + resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY] + dashboards_directory = os.path.join(resources_directory, "dashboards") os_path_exists_calls = [call('/tmp/myextension.tar.gz'), call('mpacks/myextension-ambari-mpack-1.0.0.0/mpack.json'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks'), call('/var/lib/ambari-server/resources/extensions'), call('/var/lib/ambari-server/resources/common-services'), @@ -424,6 +448,7 @@ class TestMpacks(TestCase): call(mpacks_directory + '/cache'), call('/var/lib/ambari-server/resources/dashboards'), call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/extensions'), call('/var/lib/ambari-server/resources/extensions/MYEXTENSION'), call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/extensions/MYEXTENSION/1.0/services'), @@ -476,9 +501,9 @@ class TestMpacks(TestCase): expand_mpack_mock.return_value = "mpacks/myservice-ambari-mpack-1.0.0.0" get_ambari_version_mock.return_value = "2.4.0.0" - os_path_exists_mock.side_effect = [True, True, True, True, True, True, - True, True, False, False, False, False, - True, True, True, False, True, True, + os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, + True, True, False, False, True, False, False, + True, True, True, True, False, True, True, True, False] get_ambari_properties_mock.return_value = configs @@ -491,11 +516,13 @@ class TestMpacks(TestCase): common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] mpacks_staging_directory = os.path.join(mpacks_directory, "myservice-ambari-mpack-1.0.0.0") - dashboards_directory = serverConfiguration.get_dashboard_location(configs) + resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY] + dashboards_directory = os.path.join(resources_directory, "dashboards") os_path_exists_calls = [call('/tmp/myservice.tar.gz'), call('mpacks/myservice-ambari-mpack-1.0.0.0/mpack.json'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks'), call('/var/lib/ambari-server/resources/extensions'), call('/var/lib/ambari-server/resources/common-services'), @@ -503,8 +530,10 @@ class TestMpacks(TestCase): call(mpacks_directory + '/cache'), call('/var/lib/ambari-server/resources/dashboards'), call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/common-services/MYSERVICE'), call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/common-services/MYSERVICE/1.0.0/dashboards'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks/MYSTACK'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'), @@ -562,14 +591,14 @@ class TestMpacks(TestCase): get_ambari_version_mock.return_value = "2.4.0.0" run_os_command_mock.return_value = (0, "", "") mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] - os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, - True, True, True, True, True, True, True, False, - False, True, False, False, True, False, False, - False, False, False, True, True, True, False, - True, True, False, True, True, False, False, - False, False, False, True, True, True, True, - True, True, True, False, True, False, True, True, - True, True, True, True] + os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, True, True, + True, True, True, True, True, True, True, False, False, True, + True, False, False, True, False, False, False, False, False, True, + True, True, True, False, True, True, False, True, True, False, + False, False, False, False, True, True, True, True, True, True, + True, False, True, False, True, True, True, True, True, True, + True] + get_ambari_properties_mock.return_value = configs shutil_move_mock.return_value = True @@ -579,10 +608,12 @@ class TestMpacks(TestCase): common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY] mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.1") - dashboards_directory = serverConfiguration.get_dashboard_location(configs) + resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY] + dashboards_directory = os.path.join(resources_directory, "dashboards") os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'), call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'), + call('/var/lib/ambari-server/resources'), call(mpacks_directory), call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'), call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'), @@ -591,6 +622,7 @@ class TestMpacks(TestCase): call('/tmp/mystack-1.0.0.1.tar.gz'), call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'), call('mpacks/mystack-ambari-mpack-1.0.0.1/hooks/before_upgrade.py'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks'), call('/var/lib/ambari-server/resources/extensions'), call('/var/lib/ambari-server/resources/common-services'), @@ -598,6 +630,7 @@ class TestMpacks(TestCase): call(mpacks_directory + '/cache'), call('/var/lib/ambari-server/resources/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/common-services/SERVICEA'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/1.0/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/2.0/dashboards'), @@ -607,6 +640,7 @@ class TestMpacks(TestCase): call('/var/lib/ambari-server/resources/common-services/SERVICEC'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/1.0.0/dashboards'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/2.0.0/dashboards'), + call('/var/lib/ambari-server/resources'), call('/var/lib/ambari-server/resources/stacks/MYSTACK'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'), call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'), @@ -631,6 +665,7 @@ class TestMpacks(TestCase): call('/var/lib/ambari-server/resources/dashboards/grafana-dashboards/SERVICEC'), call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards/service-metrics/SERVICEC.txt'), call('/var/lib/ambari-server/resources/dashboards/service-metrics/SERVICEC.txt'), + call('/var/lib/ambari-server/resources'), call(mpacks_directory), call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'), call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'), @@ -775,7 +810,7 @@ class TestMpacks(TestCase): serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : os.path.join(uninstall_directory, "common-services"), serverConfiguration.EXTENSION_PATH_PROPERTY : os.path.join(uninstall_directory, "extensions"), serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory, - serverConfiguration.DASHBOARD_PATH_PROPERTY : os.path.join(uninstall_directory, "dashboards"), + serverConfiguration.RESOURCES_DIR_PROPERTY : uninstall_directory, serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp" } @@ -784,7 +819,8 @@ class TestMpacks(TestCase): stacks_directory = fake_configs[serverConfiguration.STACK_LOCATION_KEY] extension_directory = fake_configs[serverConfiguration.EXTENSION_PATH_PROPERTY] common_services_directory = fake_configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] - dashboard_directory = fake_configs[serverConfiguration.DASHBOARD_PATH_PROPERTY] + resources_directory = fake_configs[serverConfiguration.RESOURCES_DIR_PROPERTY] + dashboards_directory = os.path.join(resources_directory, "dashboards") _uninstall_mpack("mystack-ambari-mpack", "1.0.0.1") @@ -794,8 +830,8 @@ class TestMpacks(TestCase): call(os.path.join(stacks_directory, "2.0/files/metainfo2.xml")), call(os.path.join(extension_directory, "SERVICEB")), call(os.path.join(common_services_directory, "SERVICEB")), - call(os.path.join(dashboard_directory, "SERVICEB")), - call(os.path.join(dashboard_directory, "files/STORM.txt"))] + call(os.path.join(dashboards_directory, "SERVICEB")), + call(os.path.join(dashboards_directory, "files/STORM.txt"))] def _create_empty_options_mock(self): options = MagicMock() http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/resources/dashboards/README.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/dashboards/README.txt b/ambari-server/src/test/resources/dashboards/README.txt new file mode 100644 index 0000000..919d510 --- /dev/null +++ b/ambari-server/src/test/resources/dashboards/README.txt @@ -0,0 +1,18 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Test directory for sync'ing a components custom folder - AMBARI-19018 http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml index f4195a8..7629552 100644 --- a/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml @@ -57,6 +57,7 @@ <name>DATANODE</name> <category>SLAVE</category> <cardinality>1+</cardinality> + <customFolder>dashboards</customFolder> <commandScript> <script>scripts/datanode.py</script> <scriptType>PYTHON</scriptType>