Repository: ambari Updated Branches: refs/heads/trunk e0ef7b3e7 -> 8d83ea62c
AMBARI-17507. Provide an option not to log ambari-agent command output for user custom script action (smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8d83ea62 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8d83ea62 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8d83ea62 Branch: refs/heads/trunk Commit: 8d83ea62c689d556dd104fcb624ca4537a796b53 Parents: e0ef7b3 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Fri Jul 1 22:51:08 2016 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Fri Jul 1 22:51:41 2016 -0700 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/ActionQueue.py | 36 +++++++----- .../test/python/ambari_agent/TestActionQueue.py | 62 ++++++++++++++++++++ .../ambari/server/agent/ExecutionCommand.java | 1 + .../AmbariCustomCommandExecutionHelper.java | 5 ++ .../AmbariManagementControllerTest.java | 3 + 5 files changed, 92 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8d83ea62/ambari-agent/src/main/python/ambari_agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py index 60c72af..3832b81 100644 --- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py +++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py @@ -279,6 +279,10 @@ class ActionQueue(threading.Thread): retryDuration = 0 # even with 0 allow one attempt retryAble = False delay = 1 + log_command_output = True + if 'commandParams' in command and 'log_output' in command['commandParams'] and "false" == command['commandParams']['log_output']: + log_command_output = False + if 'commandParams' in command: if 'max_duration_for_retries' in command['commandParams']: retryDuration = int(command['commandParams']['max_duration_for_retries']) @@ -346,21 +350,23 @@ class ActionQueue(threading.Thread): 'status': status, }) - if self.config.has_option("logging","log_command_executes") and int(self.config.get("logging", - "log_command_executes")) == 1: - if roleResult['stdout'] != '': - logger.info("Begin command output log for command with id = " + str(command['taskId']) + ", role = " - + command['role'] + ", roleCommand = " + command['roleCommand']) - self.log_command_output(roleResult['stdout'], str(command['taskId'])) - logger.info("End command output log for command with id = " + str(command['taskId']) + ", role = " - + command['role'] + ", roleCommand = " + command['roleCommand']) - - if roleResult['stderr'] != '': - logger.info("Begin command stderr log for command with id = " + str(command['taskId']) + ", role = " - + command['role'] + ", roleCommand = " + command['roleCommand']) - self.log_command_output(roleResult['stderr'], str(command['taskId'])) - logger.info("End command stderr log for command with id = " + str(command['taskId']) + ", role = " - + command['role'] + ", roleCommand = " + command['roleCommand']) + if self.config.has_option("logging","log_command_executes") \ + and int(self.config.get("logging", "log_command_executes")) == 1 \ + and log_command_output: + + if roleResult['stdout'] != '': + logger.info("Begin command output log for command with id = " + str(command['taskId']) + ", role = " + + command['role'] + ", roleCommand = " + command['roleCommand']) + self.log_command_output(roleResult['stdout'], str(command['taskId'])) + logger.info("End command output log for command with id = " + str(command['taskId']) + ", role = " + + command['role'] + ", roleCommand = " + command['roleCommand']) + + if roleResult['stderr'] != '': + logger.info("Begin command stderr log for command with id = " + str(command['taskId']) + ", role = " + + command['role'] + ", roleCommand = " + command['roleCommand']) + self.log_command_output(roleResult['stderr'], str(command['taskId'])) + logger.info("End command stderr log for command with id = " + str(command['taskId']) + ", role = " + + command['role'] + ", roleCommand = " + command['roleCommand']) if roleResult['stdout'] == '': roleResult['stdout'] = 'None' http://git-wip-us.apache.org/repos/asf/ambari/blob/8d83ea62/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py index 4f2da54..1805c9a 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py +++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py @@ -171,6 +171,22 @@ class TestActionQueue(TestCase): 'hostLevelParams':{'custom_command': 'RESTART', 'clientsToUpdateConfigs': []} } + datanode_restart_command_no_logging = { + 'commandType': 'EXECUTION_COMMAND', + 'role': u'DATANODE', + 'roleCommand': u'CUSTOM_COMMAND', + 'commandId': '1-1', + 'taskId': 9, + 'clusterName': u'cc', + 'serviceName': u'HDFS', + 'configurations': {'global': {}}, + 'configurationTags': {'global': {'tag': 'v123'}}, + 'commandParams': { + 'log_output': 'false' + }, + 'hostLevelParams': {'custom_command': 'RESTART', 'clientsToUpdateConfigs': []} + } + datanode_restart_command_no_clients_update = { 'commandType': 'EXECUTION_COMMAND', 'role': u'DATANODE', @@ -383,6 +399,52 @@ class TestActionQueue(TestCase): self.assertEqual(expected, report['reports'][0]) + @patch.object(ActionQueue, "log_command_output") + @patch.object(OSCheck, "os_distribution", new=MagicMock(return_value=os_distro_value)) + @patch.object(CustomServiceOrchestrator, "runCommand") + @patch("CommandStatusDict.CommandStatusDict") + @patch.object(ActionQueue, "status_update_callback") + def test_do_not_log_execution_commands(self, status_update_callback_mock, + command_status_dict_mock, + cso_runCommand_mock, mock_log_command_output): + custom_service_orchestrator_execution_result_dict = { + 'stdout': 'out', + 'stderr': 'stderr', + 'structuredOut': '', + 'exitcode': 0 + } + cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict + + config = AmbariConfig() + tempdir = tempfile.gettempdir() + config.set('agent', 'prefix', tempdir) + config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache") + config.set('agent', 'tolerate_download_failures', "true") + config.set('logging', 'log_command_executes', 1) + dummy_controller = MagicMock() + actionQueue = ActionQueue(config, dummy_controller) + actionQueue.execute_command(self.datanode_restart_command_no_logging) + report = actionQueue.result() + expected = {'status': 'COMPLETED', + 'configurationTags': {'global': {'tag': 'v123'}}, + 'stderr': 'stderr', + 'stdout': 'out\n\nCommand completed successfully!\n', + 'clusterName': u'cc', + 'structuredOut': '""', + 'roleCommand': u'CUSTOM_COMMAND', + 'serviceName': u'HDFS', + 'role': u'DATANODE', + 'actionId': '1-1', + 'taskId': 9, + 'customCommand': 'RESTART', + 'exitCode': 0} + # Agent caches configurationTags if custom_command RESTART completed + mock_log_command_output.assert_not_called( + [call("out\n\nCommand completed successfully!\n", "9"), call("stderr", "9")], any_order=True) + self.assertEqual(len(report['reports']), 1) + self.assertEqual(expected, report['reports'][0]) + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch("__builtin__.open") @patch.object(ActionQueue, "status_update_callback") http://git-wip-us.apache.org/repos/asf/ambari/blob/8d83ea62/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index bdb5fb1..f3f449d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -374,6 +374,7 @@ public class ExecutionCommand extends AgentCommand { String COMMAND_RETRY_ENABLED = "command_retry_enabled"; String AGENT_STACK_RETRY_ON_UNAVAILABILITY = "agent_stack_retry_on_unavailability"; String AGENT_STACK_RETRY_COUNT = "agent_stack_retry_count"; + String LOG_OUTPUT = "log_output"; /** * Comma separated list of config-types whose tags have be refreshed http://git-wip-us.apache.org/repos/asf/ambari/blob/8d83ea62/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 8bb6225..fca66db 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -1005,6 +1005,11 @@ public class AmbariCustomCommandExecutionHelper { extraParams.put(KeyNames.MAX_DURATION_OF_RETRIES, commandRetryDuration); } + // If command needs to explicitly disable STDOUT/STDERR logging + if (requestParams.containsKey(KeyNames.LOG_OUTPUT)) { + extraParams.put(KeyNames.LOG_OUTPUT, requestParams.get(KeyNames.LOG_OUTPUT)); + } + if(requestParams.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){ actionExecutionContext.getParameters().put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, requestParams.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d83ea62/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 4aa5ea4..89e3284 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -6676,6 +6676,7 @@ public class AmbariManagementControllerTest { Map<String, String> requestProperties = new HashMap<String, String>(); requestProperties.put(REQUEST_CONTEXT_PROPERTY, "Called from a test"); requestProperties.put("command_retry_enabled", "true"); + requestProperties.put("log_output", "false"); // Test multiple restarts List<RequestResourceFilter> resourceFilters = new ArrayList<RequestResourceFilter>(); @@ -6702,6 +6703,8 @@ public class AmbariManagementControllerTest { Assert.assertEquals("Expect max duration to be set", true, cParams.containsKey("max_duration_for_retries")); Assert.assertEquals("Expect max duration to be 600", "600", cParams.get("max_duration_for_retries")); Assert.assertEquals("Expect retry to be true", "true", cParams.get("command_retry_enabled")); + Assert.assertEquals("Expect log_output to be set", true, cParams.containsKey("log_output")); + Assert.assertEquals("Expect log_output to be false", "false", cParams.get("log_output")); if (hrc.getHostName().equals(host1) && hrc.getRole().equals(Role.DATANODE)) { expectedRestartCount++;
