Repository: ambari Updated Branches: refs/heads/trunk 183f71f90 -> 0941997b0
AMBARI-15067: Add custom commands for HAWQ to run check and clear cache(mithmatt via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0941997b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0941997b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0941997b Branch: refs/heads/trunk Commit: 0941997b0f68f1044242354c9f3f0f4b3a476c7d Parents: 183f71f Author: Jun Aoki <ja...@apache.org> Authored: Thu Feb 18 17:44:41 2016 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Thu Feb 18 17:44:41 2016 -0800 ---------------------------------------------------------------------- .../common-services/HAWQ/2.0.0/metainfo.xml | 16 +++++++ .../2.0.0/package/scripts/hawq_constants.py | 2 +- .../HAWQ/2.0.0/package/scripts/hawqmaster.py | 19 +++++++++ .../HAWQ/2.0.0/package/scripts/master_helper.py | 16 ++++--- .../HAWQ/2.0.0/package/scripts/params.py | 6 +++ .../HAWQ/2.0.0/package/templates/hawq-hosts.j2 | 22 ---------- .../HAWQ/2.0.0/package/templates/slaves.j2 | 22 ---------- ambari-web/app/messages.js | 2 + ambari-web/app/models/host_component.js | 44 ++++++++++++++------ ambari-web/app/utils/helper.js | 20 +++++---- ambari-web/app/views/main/service/item.js | 2 +- 11 files changed, 96 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml index 656d3a1..b77eb34 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml @@ -46,6 +46,22 @@ <timeout>1200</timeout> </commandScript> </customCommand> + <customCommand> + <name>HAWQ_CLEAR_CACHE</name> + <commandScript> + <script>scripts/hawqmaster.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </customCommand> + <customCommand> + <name>RUN_HAWQ_CHECK</name> + <commandScript> + <script>scripts/hawqmaster.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </customCommand> </customCommands> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py index 9fdbde5..2a671da 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py @@ -47,7 +47,7 @@ sysctl_conf_dir = "/etc/sysctl.d" # Files hawq_slaves_file = os.path.join(hawq_config_dir, "slaves") hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh") -hawq_hosts_file = "/tmp/hawq_hosts" +hawq_hosts_file = os.path.join(hawq_config_dir, "hawq_hosts") hawq_check_file = os.path.join(hawq_config_dir, "hawq_check.cnf") sysctl_suse_file = "/etc/sysctl.conf" sysctl_backup_file = "/etc/sysctl.conf.backup.{0}" http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py index d2f9ad0..3309561 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py @@ -17,7 +17,13 @@ See the License for the specific language governing permissions and limitations under the License. """ from resource_management import Script +from resource_management.core.resources.system import Execute +from resource_management.core.logger import Logger from resource_management.libraries.functions.check_process_status import check_process_status +try: + from resource_management.libraries.functions import hdp_select as hadoop_select +except ImportError: + from resource_management.libraries.functions import phd_select as hadoop_select import master_helper import common @@ -54,5 +60,18 @@ class HawqMaster(Script): def immediate_stop_hawq_service(self, env): master_helper.stop(hawq_constants.IMMEDIATE, hawq_constants.CLUSTER) + def hawq_clear_cache(self, env): + import params + from utils import exec_psql_cmd + cmd = "SELECT gp_metadata_cache_clear()" + Logger.info("Clearing HAWQ's HDFS Metadata cache ...") + exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port) + + def run_hawq_check(self, env): + Logger.info("Executing HAWQ Check ...") + Execute("source {0} && hawq check -f {1} --hadoop {2} --config {3}".format(hawq_constants.hawq_greenplum_path_file, hawq_constants.hawq_hosts_file, hadoop_select.get_hadoop_dir('home'), hawq_constants.hawq_check_file), + user=hawq_constants.hawq_user, + timeout=hawq_constants.default_exec_timeout) + if __name__ == "__main__": HawqMaster().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py index c5f891b..9482b35 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py @@ -18,7 +18,7 @@ limitations under the License. import os import sys from resource_management.core.resources.system import File, Execute -from resource_management.core.source import Template +from resource_management.core.source import InlineTemplate from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.libraries.functions.format import format @@ -34,14 +34,14 @@ def __setup_master_specific_conf_files(): """ import params - File(hawq_constants.hawq_check_file, content=params.hawq_check_content, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, - mode=0644) + params.File(hawq_constants.hawq_check_file, + content=params.hawq_check_content) - File(hawq_constants.hawq_slaves_file, content=Template("slaves.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, - mode=0644) + params.File(hawq_constants.hawq_slaves_file, + content=InlineTemplate("{% for host in hawqsegment_hosts %}{{host}}\n{% endfor %}")) - File(hawq_constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group, - mode=0644) + params.File(hawq_constants.hawq_hosts_file, + content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}")) def __setup_passwordless_ssh(): @@ -51,8 +51,6 @@ def __setup_passwordless_ssh(): import params utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password)) - File(hawq_constants.hawq_hosts_file, action='delete') - def configure_master(): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py index 74c9813..7633731 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py @@ -19,6 +19,7 @@ limitations under the License. import functools import hawq_constants from resource_management import Script +from resource_management.core.resources.system import File from resource_management.libraries.functions.default import default from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.resources.xml_config import XmlConfig @@ -75,6 +76,11 @@ HdfsResource = functools.partial(HdfsResource, hdfs_site=hdfs_site, default_fs=default_fs) +# File partial function +File = functools.partial(File, + owner=hawq_constants.hawq_user, + group=hawq_constants.hawq_group, + mode=0644) # XMLConfig partial function XmlConfig = functools.partial(XmlConfig, http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 deleted file mode 100644 index e093375..0000000 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -{% for host in hawq_all_hosts %} -{{host}} -{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 deleted file mode 100644 index 0435b3e..0000000 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -{% for host in hawqsegment_hosts %} -{{host}} -{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index 9d24b4f..1b08248 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1690,6 +1690,8 @@ Em.I18n.translations = { 'services.service.actions.run.immediateStopHawqSegment.context':'Stop HAWQ Segment (Immediate Mode)', 'services.service.actions.run.resyncHawqStandby.context':'Re-Sync HAWQ Standby', 'services.service.actions.run.resyncHawqStandby.label':'Re-Synchronize HAWQ Standby', + 'services.service.actions.run.clearHawqCache.label':'Clear HAWQ\'s HDFS Metadata Cache', + 'services.service.actions.run.runHawqCheck.label':'Run HAWQ Check', 'services.service.actions.manage_configuration_groups.short':'Manage Config Groups', 'services.service.actions.serviceActions':'Service Actions', http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-web/app/models/host_component.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js index d3d4ab9..61a51cd 100644 --- a/ambari-web/app/models/host_component.js +++ b/ambari-web/app/models/host_component.js @@ -375,6 +375,12 @@ App.HostComponentActionMap = { hasSubmenu: ctx.get('controller.isSeveralClients'), submenuOptions: ctx.get('controller.clientComponents') }, + DELETE_SERVICE: { + action: 'deleteService', + context: ctx.get('serviceName'), + label: Em.I18n.t('common.delete'), + cssClass: 'icon-remove' + }, IMMEDIATE_STOP_HAWQ_SERVICE: { action: 'executeHawqCustomCommand', customCommand: 'IMMEDIATE_STOP_HAWQ_SERVICE', @@ -398,18 +404,6 @@ App.HostComponentActionMap = { isHidden : App.get('isSingleNode') || !HS , disabled: !((!!HMComponent && HMComponent.get('startedCount') === 1) && (!!HS && HS.get('workStatus') === App.HostComponentStatus.started)) }, - MASTER_CUSTOM_COMMAND: { - action: 'executeCustomCommand', - cssClass: 'icon-play-circle', - isHidden: false, - disabled: false - }, - DELETE_SERVICE: { - action: 'deleteService', - context: ctx.get('serviceName'), - label: Em.I18n.t('common.delete'), - cssClass: 'icon-remove' - }, TOGGLE_ADD_HAWQ_STANDBY: { action: 'addHawqStandby', label: Em.I18n.t('admin.addHawqStandby.button.enable'), @@ -424,7 +418,31 @@ App.HostComponentActionMap = { cssClass: 'icon-arrow-up', isHidden: App.get('isSingleNode') || !HS, disabled: false - } + }, + HAWQ_CLEAR_CACHE: { + action: 'executeHawqCustomCommand', + customCommand: 'HAWQ_CLEAR_CACHE', + context: Em.I18n.t('services.service.actions.run.clearHawqCache.label'), + label: Em.I18n.t('services.service.actions.run.clearHawqCache.label'), + cssClass: 'icon-refresh', + isHidden : false, + disabled: !HM || HM.get('workStatus') != App.HostComponentStatus.started + }, + RUN_HAWQ_CHECK: { + action: 'executeHawqCustomCommand', + customCommand: 'RUN_HAWQ_CHECK', + context: Em.I18n.t('services.service.actions.run.runHawqCheck.label'), + label: Em.I18n.t('services.service.actions.run.runHawqCheck.label'), + cssClass: 'icon-thumbs-up-alt', + isHidden : false, + disabled: false + }, + MASTER_CUSTOM_COMMAND: { + action: 'executeCustomCommand', + cssClass: 'icon-play-circle', + isHidden: false, + disabled: false + } }; } }; http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-web/app/utils/helper.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/helper.js b/ambari-web/app/utils/helper.js index 8f4985f..ff5fe11 100644 --- a/ambari-web/app/utils/helper.js +++ b/ambari-web/app/utils/helper.js @@ -661,16 +661,22 @@ App.format = { } // HAWQ custom commands on back Ops page. if (result === ' Resync Hawq Standby HAWQ Standby Master') { - result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label'); - } + result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label'); + } if (result === ' Immediate Stop Hawq Service HAWQ Master') { - result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label'); - } + result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label'); + } if (result === ' Immediate Stop Hawq Segment HAWQ Segment') { - result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label'); - } + result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label'); + } if(result === ' Activate Hawq Standby HAWQ Standby Master') { - result = Em.I18n.t('admin.activateHawqStandby.button.enable'); + result = Em.I18n.t('admin.activateHawqStandby.button.enable'); + } + if(result === ' Hawq Clear Cache HAWQ Master') { + result = Em.I18n.t('services.service.actions.run.clearHawqCache.label'); + } + if(result === ' Run Hawq Check HAWQ Master') { + result = Em.I18n.t('services.service.actions.run.runHawqCheck.label'); } //<---End HAWQ custom commands---> return result; http://git-wip-us.apache.org/repos/asf/ambari/blob/0941997b/ambari-web/app/views/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js index e74a588..020048b 100644 --- a/ambari-web/app/views/main/service/item.js +++ b/ambari-web/app/views/main/service/item.js @@ -35,7 +35,7 @@ App.MainServiceItemView = Em.View.extend({ 'RESOURCEMANAGER': ['DECOMMISSION', 'REFRESHQUEUES'], 'HBASE_MASTER': ['DECOMMISSION'], 'KNOX_GATEWAY': ['STARTDEMOLDAP','STOPDEMOLDAP'], - 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE'], + 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE'], 'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'], 'HAWQSTANDBY' : ['RESYNC_HAWQ_STANDBY','ACTIVATE_HAWQ_STANDBY'] },
