Repository: ambari Updated Branches: refs/heads/trunk 7259d972e -> 424cca6c3
AMBARI-14855: Add Alert for HAWQSTANDBY sync status with HAWQMASTER (mithmatt via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/424cca6c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/424cca6c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/424cca6c Branch: refs/heads/trunk Commit: 424cca6c3744eca2c44dff0fbc47791492845f88 Parents: 7259d97 Author: Jun Aoki <ja...@apache.org> Authored: Wed Feb 3 10:58:56 2016 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Wed Feb 3 10:58:56 2016 -0800 ---------------------------------------------------------------------- .../common-services/HAWQ/2.0.0/alerts.json | 19 ++ .../2.0.0/package/alerts/alert_sync_status.py | 91 +++++++++ .../stacks/2.3/HAWQ/test_alert_sync_status.py | 194 +++++++++++++++++++ 3 files changed, 304 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/424cca6c/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/alerts.json b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/alerts.json new file mode 100644 index 0000000..3119a0c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/alerts.json @@ -0,0 +1,19 @@ +{ + "HAWQ": { + "HAWQMASTER": [ + { + "name": "hawqstandby_sync_status", + "label": "HAWQ Standby Sync Status", + "description": "This alert will trigger if HAWQ Standby is not synchronized with HAWQ Master", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HAWQ/2.0.0/package/alerts/alert_sync_status.py", + "parameters": [] + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/424cca6c/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/alerts/alert_sync_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/alerts/alert_sync_status.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/alerts/alert_sync_status.py new file mode 100644 index 0000000..c94be9e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/alerts/alert_sync_status.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +from resource_management.core.shell import call + +HAWQ_USER = 'gpadmin' +HAWQ_GREENPLUM_PATH_FILE = '/usr/local/hawq/greenplum_path.sh' + +HAWQMASTER_PORT = '{{hawq-site/hawq_master_address_port}}' +HAWQSTANDBY_ADDRESS = '{{hawq-site/hawq_standby_address_host}}' + +RESULT_STATE_OK = 'OK' +RESULT_STATE_WARNING = 'WARNING' +RESULT_STATE_UNKNOWN = 'UNKNOWN' +RESULT_STATE_SKIPPED = 'SKIPPED' + +logger = logging.getLogger('ambari_alerts') + + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used to build the dictionary passed into execute + """ + return (HAWQMASTER_PORT, HAWQSTANDBY_ADDRESS) + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_STATE_UNKNOWN, ['There were no configurations supplied to the script.']) + + # If HAWQSTANDBY is not installed on the cluster + if HAWQSTANDBY_ADDRESS not in configurations: + return (RESULT_STATE_SKIPPED, ['HAWQSTANDBY is not installed.']) + + try: + sync_status = get_sync_status(configurations[HAWQMASTER_PORT]) + if sync_status in ('Synchronized', 'Synchronizing'): + return (RESULT_STATE_OK, ['HAWQSTANDBY is in sync with HAWQMASTER.']) + elif sync_status == 'Not Synchronized': + return (RESULT_STATE_WARNING, ['HAWQSTANDBY is not in sync with HAWQMASTER.']) + except Exception, e: + logger.exception('[Alert] Retrieving HAWQSTANDBY sync status from HAWQMASTER fails on host, {0}:'.format(host_name)) + logger.exception(str(e)) + + # Sync status cannot be determined + return (RESULT_STATE_UNKNOWN, ['Sync status cannot be determined.']) + + +def get_sync_status(port): + """ + Gets the sync status of HAWQSTANDBY from HAWQMASTER by running a SQL command. + summary_state can be of the following values: ('Synchronized', 'Synchronizing', 'Not Synchronized', 'None', 'Not Configured', 'Unknown') + """ + query = "SELECT summary_state FROM gp_master_mirroring" + cmd = "source {0} && psql -p {1} -t -d template1 -c \"{2};\"".format(HAWQ_GREENPLUM_PATH_FILE, port, query) + + returncode, output = call(cmd, + user=HAWQ_USER, + timeout=60) + + if returncode: + raise + + return output.strip() http://git-wip-us.apache.org/repos/asf/ambari/blob/424cca6c/ambari-server/src/test/python/stacks/2.3/HAWQ/test_alert_sync_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_alert_sync_status.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_alert_sync_status.py new file mode 100644 index 0000000..7d030dc --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_alert_sync_status.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +# System imports +import os +import sys + +from mock.mock import patch + +# Local imports +from stacks.utils.RMFTestCase import * + +COMMON_SERVICES_ALERTS_DIR = "HAWQ/2.0.0/package/alerts" + +file_path = os.path.dirname(os.path.abspath(__file__)) +file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) +file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_ALERTS_DIR) + +RESULT_STATE_OK = 'OK' +RESULT_STATE_WARNING = 'WARNING' +RESULT_STATE_UNKNOWN = 'UNKNOWN' +RESULT_STATE_SKIPPED = 'SKIPPED' + +class TestAlertSyncStatus(RMFTestCase): + + def setUp(self): + """ + Import the class under test. + Because the class is present in a different folder, append its dir to the system path. + Also, shorten the import name and make it a global so the test functions can access it. + :return: + """ + sys.path.append(file_path) + global alert_sync_status + import alert_sync_status + + def test_missing_configs(self): + """ + Check that the status is UNKNOWN when configs are missing. + """ + configs = None + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'There were no configurations supplied to the script.') + + + @patch("alert_sync_status.get_sync_status") + def test_no_standby_state(self, get_sync_status_mock): + """ + Test that the status is SKIPPED when HAWQSTANDBY is not in configurations + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432" + } + + # Mock calls + get_sync_status_mock.return_value = 'Not Configured' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_SKIPPED) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is not installed.') + + + @patch("alert_sync_status.get_sync_status") + def test_synchronized_state(self, get_sync_status_mock): + """ + Test that the status is OK when HAWQSTANDBY is 'Synchronized' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Synchronized' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is in sync with HAWQMASTER.') + + + @patch("alert_sync_status.get_sync_status") + def test_synchronizing_state(self, get_sync_status_mock): + """ + Test that the status is OK when HAWQSTANDBY is 'Synchronizing' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Synchronizing' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_OK) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is in sync with HAWQMASTER.') + + + @patch("alert_sync_status.get_sync_status") + def test_not_synchronized_state(self, get_sync_status_mock): + """ + Test that the status is WARNING when HAWQSTANDBY is 'Noe Synchronized' with HAWQMASTER + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Not Synchronized' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_WARNING) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'HAWQSTANDBY is not in sync with HAWQMASTER.') + + + @patch("alert_sync_status.get_sync_status") + def test_none_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'None' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'None' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.') + + + @patch("alert_sync_status.get_sync_status") + def test_not_configured_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'Not Configured' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Not Configured' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.') + + + @patch("alert_sync_status.get_sync_status") + def test_unknown_state(self, get_sync_status_mock): + """ + Test that the status is UNKNOWN when HAWQMASTER returns summary_state as 'Unknown' + """ + configs = { + "{{hawq-site/hawq_master_address_port}}": "5432", + "{{hawq-site/hawq_standby_address_host}}": "c6402.ambari.apache.org" + } + + # Mock calls + get_sync_status_mock.return_value = 'Unknown' + + [status, messages] = alert_sync_status.execute(configurations=configs) + self.assertEqual(status, RESULT_STATE_UNKNOWN) + self.assertTrue(messages is not None and len(messages) == 1) + self.assertEqual(messages[0], 'Sync status cannot be determined.')