AMBARI-21420 HBase master crashed during/post EU on an IOP4.2.5/HDP2.6 migrated cluster (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1f54c6e2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1f54c6e2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1f54c6e2 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 1f54c6e272a5a2ad176619062b31ca18bbdf93ea Parents: 83761d4 Author: Di Li <d...@apache.org> Authored: Wed Jul 12 15:59:35 2017 -0400 Committer: Di Li <d...@apache.org> Committed: Wed Jul 12 15:59:35 2017 -0400 ---------------------------------------------------------------------- .../0.96.0.2.0/package/scripts/hbase_service.py | 17 +++++++++++++++++ .../0.96.0.2.0/package/scripts/params_linux.py | 9 +++++++++ .../BigInsights/4.2.5/upgrades/config-upgrade.xml | 11 +++++++++++ .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 5 +++++ .../BigInsights/4.2/upgrades/config-upgrade.xml | 11 +++++++++++ .../4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 7 ++++++- 6 files changed, 59 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py index a1003dc..3b8e494 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_service.py @@ -19,6 +19,7 @@ limitations under the License. """ from resource_management import * +from resource_management.core.logger import Logger def hbase_service( name, @@ -32,6 +33,22 @@ def hbase_service( pid_expression = as_sudo(["cat", pid_file]) no_op_test = as_sudo(["test", "-f", pid_file]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1") + # delete wal log if HBase version has moved down + if params.to_backup_wal_dir: + wal_directory = params.wal_directory + timestamp = datetime.datetime.now() + timestamp_format = '%Y%m%d%H%M%S' + wal_directory_backup = '%s_%s' % (wal_directory, timestamp.strftime(timestamp_format)) + + rm_cmd = format("hadoop fs -mv {wal_directory} {wal_directory_backup}") + try: + Execute ( rm_cmd, + user = params.hbase_user + ) + except Exception, e: + #Should still allow HBase Start/Stop to proceed + Logger.error("Failed to backup HBase WAL directory, command: {0} . Exception: {1}".format(rm_cmd, e.message)) + if action == 'start': daemon_cmd = format("{cmd} start {role}") http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 1ee5248..6617a80 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -44,6 +44,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions.expect import expect from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config +from resource_management.libraries.functions.constants import Direction # server configurations config = Script.get_config() @@ -440,3 +441,11 @@ if has_atlas: atlas_with_managed_hbase = len(zk_hosts_matches) > 0 else: atlas_with_managed_hbase = False + +wal_directory = "/apps/hbase/data/MasterProcWALs" + +backup_wal_dir = default('/configurations/hbase-env/backup_wal_dir', False) + +#Need to make sure not to keep removing WAL logs once EU is finalized. +upgrade_direction = default("/commandParams/upgrade_direction", None) +to_backup_wal_dir = upgrade_direction is not None and upgrade_direction == Direction.UPGRADE and backup_wal_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml index 42999b2..b51a744 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml @@ -52,6 +52,17 @@ </component> </service> + <service name="HBASE"> + <component name="HBASE_MASTER"> + <changes> + <definition xsi:type="configure" id="biginsights_4_2_hbase_env_config" summary="Update HBase configurations"> + <type>hbase-env</type> + <set key="backup_wal_dir" value="true"/> + </definition> + </changes> + </component> + </service> + <service name="HIVE"> <component name="HIVE_SERVER"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index a96ede9..f3c73a0 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -182,6 +182,11 @@ <task xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch" /> </execute-stage> + <!-- HBASE --> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase"> + <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" /> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_0_0_hive_env_configure"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index f9e3e15..b46f476 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -52,6 +52,17 @@ </component> </service> + <service name="HBASE"> + <component name="HBASE_MASTER"> + <changes> + <definition xsi:type="configure" id="biginsights_4_2_hbase_env_config" summary="Update HBase configurations"> + <type>hbase-env</type> + <set key="backup_wal_dir" value="true"/> + </definition> + </changes> + </component> + </service> + <service name="HIVE"> <component name="HIVE_SERVER"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1f54c6e2/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index a96ede9..4867626 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -181,7 +181,12 @@ <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer"> <task xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch" /> </execute-stage> - + + <!-- HBASE --> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase"> + <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" /> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_0_0_hive_env_configure"/>
