Alex Lourie has uploaded a new change for review. Change subject: [WIP] packaging: added DB validation functions on upgrade ......................................................................
[WIP] packaging: added DB validation functions on upgrade Change-Id: I25979acbf54d980168be929638ff4aed800bf6d3 Signed-off-by: Alex Lourie <[email protected]> --- M packaging/setup/ovirt_engine_setup/constants.py A packaging/setup/plugins/ovirt-engine-setup/upgrade/__init__.py A packaging/setup/plugins/ovirt-engine-setup/upgrade/asynctasks.py A packaging/setup/plugins/ovirt-engine-setup/upgrade/dbvalidations.py 4 files changed, 636 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/70/15970/1 diff --git a/packaging/setup/ovirt_engine_setup/constants.py b/packaging/setup/ovirt_engine_setup/constants.py index 1009c0e..8bc8536 100644 --- a/packaging/setup/ovirt_engine_setup/constants.py +++ b/packaging/setup/ovirt_engine_setup/constants.py @@ -124,6 +124,20 @@ '.pgpass', ) + OVIRT_ENGINE_DB_UTILS_DIR = os.path.join( + OVIRT_ENGINE_DATADIR, + 'scripts', + 'dbutils' + ) + OVIRT_ENGINE_DB_VALIDATOR = os.path.join( + OVIRT_ENGINE_DB_UTILS_DIR, + 'validatedb.sh' + ) + OVIRT_ENGINE_TASKCLEANER = os.path.join( + OVIRT_ENGINE_DB_UTILS_DIR, + 'taskcleaner.sh' + ) + OVIRT_ENGINE_DB_DIR = os.path.join( OVIRT_ENGINE_DATADIR, 'dbscripts', @@ -441,6 +455,7 @@ DB_CREDENTIALS_AVAILABLE = 'osetup.db.connection.credentials' DB_CONNECTION_AVAILABLE = 'osetup.db.connection.available' DB_SCHEMA = 'osetup.db.schema' + FIX_DB_VIOLATIONS = 'osetup.db.fix.violations' NET_FIREWALL_MANAGER_AVAILABLE = 'osetup.net.firewallmanager.available' NET_FIREWALL_MANAGER_PROCESS_TEMPLATES = \ 'osetup.net.firewallmanager.templates.available' @@ -634,6 +649,17 @@ def REMOVE_EMPTY_DATABASE(self): return 'OVESETUP_DB/cleanupRemove' + @osetupattrs( + answerfile=True, + ) + def FIX_DB_VIOLATIONS(self): + return 'OVESETUP_DB/fixDbViolations' + f + @osetupattrs( + answerfile=True, + ) + def IGNORE_TASKS(self): + return 'OVESETUP_DB/ignoreTasks' @util.export @util.codegen diff --git a/packaging/setup/plugins/ovirt-engine-setup/upgrade/__init__.py b/packaging/setup/plugins/ovirt-engine-setup/upgrade/__init__.py new file mode 100644 index 0000000..8050687 --- /dev/null +++ b/packaging/setup/plugins/ovirt-engine-setup/upgrade/__init__.py @@ -0,0 +1,35 @@ +# +# ovirt-engine-setup -- ovirt engine setup +# Copyright (C) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""ovirt-host-setup upgrade validations plugin.""" + + +from otopi import util + + +from . import dbvalidations +from . import asynctasks + + [email protected] +def createPlugins(context): + dbvalidations.Plugin(context=context) + asynctasks.Pluging(context=context) + + +# vim: expandtab tabstop=4 shiftwidth=4 diff --git a/packaging/setup/plugins/ovirt-engine-setup/upgrade/asynctasks.py b/packaging/setup/plugins/ovirt-engine-setup/upgrade/asynctasks.py new file mode 100644 index 0000000..b4cf7f9 --- /dev/null +++ b/packaging/setup/plugins/ovirt-engine-setup/upgrade/asynctasks.py @@ -0,0 +1,383 @@ +# +# ovirt-engine-setup -- ovirt engine setup +# Copyright (C) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" DB Async tasks handling plugin.""" + + +import datetime +import gettext +_ = lambda m: gettext.dgettext(message=m, domain='ovirt-engine-setup') + + +from otopi import util +from otopi import plugin + + +from ovirt_engine_setup import constants as osetupcons +from ovirt_engine_setup import dialog + + [email protected] +class Plugin(plugin.PluginBase): + """ DB Async tasks handling plugin.""" + + _CLEANUP_WAIT_SECONDS = 180 + _CLEANUP_WAIT_MINUTES = self._CLEANUP_WAIT_SECONDS / 60 + + class _engineInMaintenance(object): + def __init__( + self, + configureTasksTimeout, + setEngineMode, + services, + logger, + ): + self._configureTasksTimeout = configureTasksTimeout + self._setEngineMode = setEngineMode + self.services = services + self.logger = logger + self.timeout = 0 + + def __enter__(self): + self.timeout = self._configureTasksTimeout( + timeout=self.timeout + ) + self.logger.debug( + 'Setting engine into maintenance mode' + ) + self._setEngineMode(mode='maintenance') + + # Start the engine in maintenance mode + self.services.state( + name=self.environment[ + osetupcons.Const.ENGINE_SERVICE_NAME + ], + state=True, + ) + + def __exit__(self, exc_type, exc_value, traceback): + # Restore previous engine configuration + self.logger.debug( + 'Restoring engine normal mode' + ) + # Stop the engine first + self.services.state( + name=self.environment[ + osetupcons.Const.ENGINE_SERVICE_NAME + ], + state=False, + ) + # Restore normal mode + self._setEngineMode(mode='normal') + # Restore previous zombie timeout + self._configureTasksTimeout( + timeout=self.timeout + ) + + def __init__(self, context): + super(Plugin, self).__init__(context=context) + self._enabled = False + + def _zombieUtil(self, dbname, user, host, port, clear=None): + + args = [ + osetupcons.FileLocations.OVIRT_ENGINE_TASKCLEANER, + '--user={user}'.format( + user=user, + ), + '--host={host}'.format( + host=host + ), + '--port={port}'.format( + port=port + ), + '--database={database}'.format( + database=dbname + ), + ] + if clear: + args.append = [ + '-R', + '-A', + '-J', + '-q', + ] + + return self.execute( + args, + raiseOnError=False, + ) + + + def _clearZombieTasks(self): + rc, tasks, stderr = self._zombieUtil( + dbname=self.environment[ + osetupcons.DBEnv.DATABASE + ], + user=self.environment[ + osetupcons.DBEnv.USER + ], + host=self.environment[ + osetupcons.DBEnv.HOST + ], + port=self.environment[ + osetupcons.DBEnv.PORT + ], + clear=True, + ) + if rc > 1: + raise RuntimeError( + _('Failed to clear zombie tasks!') + ) + + def _setEngineMode(self, mode='normal'): + if mode == 'maintenace': + self.dbstatement.updateVDCOption( + { + 'name': 'EngineMode', + 'value': 'MAINTENANCE', + } + ) + self.in_maintenance = True + elif mode == 'normal': + self.dbstatement.updateVDCOption( + { + 'name': 'EngineMode', + 'value': 'ACTIVE', + } + ) + self.in_maintenance = False + else: + self.logger.debug( + 'Usupported Engine mode requested, doing nothing.' + ) + + + def _configureTasksTimeout(self, timeout): + + # First, get the originalTimeout value + originalTimeout = self.dbstatement.getVDCOption( + name='AsyncTaskZombieTaskLifeInMinutes' + ) + + # Now, set the value to timeout, it raises an Exception if it fails + self.dbstatement.updateVDCOption( + { + 'name': 'AsyncTaskZombieTaskLifeInMinutes', + 'value': timeout, + } + ) + + # If everything went fine, return the original value + return originalTimeout + + + def _getRunningTasks(self, dbname, user, host, port, password): + + tasks = self.dbstatement.execute( + statement=""" + select + a.action_type, + a.task_id, + a.started_at, + b.name + from async_tasks a, storage_pool b + where a.storage_pool_id = b.id; + """, + )[0] + + from async_tasks_map import async_tasks_map + return '\n'.join( + [ + '\n---- Task ID: {task_id:30} ------- \n' + 'Task Name: {task_name:30}\n' + 'Task Description: {task_desc:30}\n' + 'Started at: {started_at:30}\n' + 'DC Name: {name:30}\n'.format( + task_id=entry['task_id'], + task_name=async_tasks_map[entry['action_type']][0], + task_desc=async_tasks_map[entry['action_type']][1], + started_at=entry['started_at'], + name=entry['name'], + ) + for entry in tasks + ] + ) + + def _getCompensations(self, dbname, user, host, port, password): + + compensations = self.dbstatement.execute( + statement=""" + select + command_type, entity_type + from business_entity_snapshot; + """, + )[0] + + return '\n'.join( + [ + '{command_type:30} {entity_type:30}'.format(**entry) + for entry in compensations + ] + ) + + def _stopTasks(self, runningTasks, compensations): + now = datetime.datetime.now() + timestamp = now.strftime('%b %d %H:%M:%S') + return self.dialog.queryBoolean( + dialog=self.dialog, + name='OVESETUP_SHOW_RUNNING_TASKS', + note=_( + '[ {timestamp} ] The following system tasks have been ' + 'found running in the system:\n' + '{tasks}\n' + '{compensations}\n' + 'Would you like to proceed ' + 'and try to stop these tasks automatically?\n' + '(Answering "no" will stop the upgrade (@VALUES@) ' + ).format( + timestamp=timestamp, + tasks=runningTasks, + compensations=compensations, + ), + prompt=True, + true=_('Yes'), + false=_('No'), + ) + + def _infoStoppingTasks(self, retry=False): + now = datetime.datetime.now() + timestamp = now.strftime('%b %d %H:%M:%S') + if retry: + header = 'Retrying to clear system tasks ' + else: + header = 'System will try to clear running tasks ' + + return _( + '[ {timestamp} ] {header}' + 'during the next {cleanup_wait_minutes} minutes.\n' + ).format( + timestamp = timestamp, + header=header, + cleanup_wait_minutes=self._CLEANUP_WAIT_MINUTES + ) + + def _clearRunningTasks(self): + + # Get Tasks + runningTasks = self._getRunningTasks() + compensations = self._getCompensations() + + if runningTasks or compensations: + + if not self._stopTasks(runningTasks, compensations): + # User decided not to stop tasks + raise RuntimeError( + _('User decided not to stop running tasks. Exiting.') + ) + + self.dialog.note( + text=self._infoStoppingTasks() + ) + + with self._engineInMaintenance( + configureTasksTimeout=self._configureTasksTimeout, + setEngineMode=self._setEngineMode, + services=self.services, + logger=self.logger, + ): + # Pull tasks in a loop for some time + # _CLEANUP_WAIT_SECONDS = 180 (seconds, between trials) + waited = 0 + while runningTasks or compensations: + time.sleep(1) + waited += 1 + runningTasks = self._getRunningTasks() + compensations = self._getCompensations() + + if runningTasks or compensations: + self.logger.debug( + 'Still waiting for system tasks to be cleared.' + ) + if waited < self._CLEANUP_WAIT_SECONDS: + continue + + # Should we continue? + if self._stopTasks(runningTasks, compensations): + #If yes, go for another iteration + self.dialog.note( + text=self._infoStoppingTasks(retry=True) + ) + waited = 0 + else: + # If not, break the loop + # There are still tasks running, so exit and tell + # to resolve before user continues. + raise RuntimeError( + _( + 'There are still running tasks: {tasks} ' + 'Please make sure that there are no ' + 'running system tasks before you ' + 'continue. Stopping upgrade.' + ).format( + tasks=_( + '\nRunning System Tasks:\n' + '{tasks}' + '{compensations}' + ).format( + tasks=runningTasks, + compensations=compensations, + ) + ) + ) + + @plugin.event( + stage=plugin.Stages.STAGE_INIT, + ) + def _init(self): + self.environment.setdefault( + osetupcons.DBEnv.IGNORE_TASKS, + False + ) + + @plugin.event( + stage=plugin.Stages.STAGE_SETUP, + ) + def _setup(self): + self._enabled = self.environment[ + osetupcons.DBEnv.IGNORE_TASKS + ] == False + self.in_maintenance = False + self.originalTimeout = 0 + + @plugin.event( + stage=plugin.Stages.STAGE_CUSTOMIZATION, + after=[ + osetupcons.Stages.FIX_DB_VIOLATIONS + ], + #before yum update + condition=lambda self: self._enabled, + ) + def _customization(self): + + self.dbstatement = database.Statement(self.environment) + self._clearZombieTasks() + self._clearRunningTasks() + + +# vim: expandtab tabstop=4 shiftwidth=4 diff --git a/packaging/setup/plugins/ovirt-engine-setup/upgrade/dbvalidations.py b/packaging/setup/plugins/ovirt-engine-setup/upgrade/dbvalidations.py new file mode 100644 index 0000000..ebf2868 --- /dev/null +++ b/packaging/setup/plugins/ovirt-engine-setup/upgrade/dbvalidations.py @@ -0,0 +1,192 @@ +# +# ovirt-engine-setup -- ovirt engine setup +# Copyright (C) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" DB validations plugin.""" + + +import gettext +_ = lambda m: gettext.dgettext(message=m, domain='ovirt-engine-setup') + + +from otopi import util +from otopi import plugin + + +from ovirt_engine_setup import constants as osetupcons +from ovirt_engine_setup import dialog + + [email protected] +class Plugin(plugin.PluginBase): + """ DB validations plugin.""" + + def _dbUtil(self, dbname, user, host, port, fix=None): + + args = [ + osetupcons.FileLocations.OVIRT_ENGINE_DB_VALIDATOR, + '--user={user}'.format( + user=user, + ), + '--host={host}'.format( + host=host + ), + '--port={port}'.format( + port=port + ), + '--database={database}'.format( + database=dbname + ), + ] + if fix: + args.append = [ + '--fix=1' + ] + + return self.execute( + args, + raiseOnError=False, + ) + + def _checkDb(self, dbname, user, host, port): + + rc, stdout, stderr = self._dbUtil( + dbname=dbname, + user=user, + host=host, + port=port, + ) + if rc != 0: + raise Exception( + 'Error: failed checking DB:\n{output}\n'.format( + output=stdout, + ) + ) + + result = ( + '\n\n ====== Validating database "{dbname}" ====== \n' + '{content}' + ).format( + dbname=dbname, + content=stdout, + ) + + return (result, rc) + + def _fixDb(self, dbname, user, host, port): + + self._dbUtil( + dbname=dbname, + user=user, + host=host, + port=port, + fix=True, + ) + + def __init__(self, context): + super(Plugin, self).__init__(context=context) + self._enabled = False + + @plugin.event( + stage=plugin.Stages.STAGE_INIT, + ) + def _init(self): + self.environment.setdefault( + osetupcons.DBEnv.FIX_DB_VIOLATIONS, + None + ) + + @plugin.event( + stage=plugin.Stages.STAGE_SETUP, + ) + def _setup(self): + self._enabled = self.environment[ + osetupcons.RPMDistroEnv.ENABLE_UPGRADE + ] + + @plugin.event( + stage=plugin.Stages.STAGE_CUSTOMIZATION, + name=osetupcons.Stages.FIX_DB_VIOLATIONS, + after=[ + osetupcons.Stages.DB_CONNECTION_CUSTOMIZATION, + ], + condition=lambda self: self._enabled, + ) + def _customization(self): + violations, issues_found = self._checkDb( + dbname=self.environment[ + osetupcons.DBEnv.DATABASE + ], + user=self.environment[ + osetupcons.DBEnv.USER + ], + host=self.environment[ + osetupcons.DBEnv.HOST + ], + port=self.environment[ + osetupcons.DBEnv.PORT + ], + ) + if issues_found: + if self.environment[ + osetupcons.DBEnv.FIX_DB_VIOLATIONS + ] is None: + self.environment[ + osetupcons.DBEnv.FIX_DB_VIOLATIONS + ] = self.dialog.queryBoolean( + dialog=self.dialog, + name='OVESETUP_FIX_DB_VALIDATIONS', + note=_( + 'The following inconsistencies were found ' + 'in the DB: {violations}. Would you like ' + 'to automatically clear inconsistencies before ' + 'upgraing?\n' + '(Answering no will stop the upgrade) (@VALUES@): ' + ).format( + violations=violations + ), + prompt=True, + true=_('Yes'), + false=_('No'), + ) + + if self.environment[ + osetupcons.DBEnv.FIX_DB_VIOLATIONS + ]: + self._fixDb( + dbname=self.environment[ + osetupcons.DBEnv.DATABASE + ], + user=self.environment[ + osetupcons.DBEnv.USER + ], + host=self.environment[ + osetupcons.DBEnv.HOST + ], + port=self.environment[ + osetupcons.DBEnv.PORT + ], + ) + else: + raise Exception( + _( + "User decided to skip db fix" + ) + ) + + +# vim: expandtab tabstop=4 shiftwidth=4 -- To view, visit http://gerrit.ovirt.org/15970 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I25979acbf54d980168be929638ff4aed800bf6d3 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Alex Lourie <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
