cfgupgrade can now upgrade the configuration to the newest format, which requires the disks_active flag for all instances. Additionally, cfgupgrade now supports the --target-version parameter, which is used during tests and has to be specified when downgrading a configuration.
Signed-off-by: Thomas Thrainer <[email protected]> --- As discussed in the ganeti-core meeting, this patch is only a RFC for cfgupgrade. cfgupgrade * now can freely upgrade/downgrade from/to 2.{6..9} (which might be interesting for users who use distribution packages, which might be upgraded from 2.6 to 2.9 directly for instance). * does still not support downgrades to 2.{0..5}, and does not check for that. * supports a --target-version parameter to specify the version to up-/downgrade to. Omitting it uses the current Ganeti version. * structures the code in such a way that future up-/downgrade steps should be rather easy to integrate. On the downside, cfgupgrade got more complex through this, and it's virtually impossible to test all up-/downgrade paths. An alternative to this patch might be to add disks_active in objects.py, NodeGroup.UpgradeConfig(). The downgrade path could either be ignored or integrated in cfgupgrade. Do you see any need to support such up-/downgrade paths? For development it comes in handy sometimes, when you are switching between branches a lot. But would there be a need for others as well? And would structuring the code in this way (having functions which upgrade just from one to the next version, which then are applied one after the other) help maintainability or should we just ignore which version changes came in, and try to upgrade to the current version? UPGRADE | 8 ++- test/py/cfgupgrade_unittest.py | 56 +++++++++++---- tools/cfgupgrade | 152 +++++++++++++++++++++++++---------------- 3 files changed, 139 insertions(+), 77 deletions(-) diff --git a/UPGRADE b/UPGRADE index a769b92..7a6e938 100644 --- a/UPGRADE +++ b/UPGRADE @@ -116,10 +116,12 @@ revert the configuration **before** installing the old version. $ tar czf /var/lib/ganeti-$(date +\%FT\%T).tar.gz -C /var/lib ganeti -#. Run cfgupgrade on the master node:: +#. Run cfgupgrade on the master node (setting <old-ver> to 2.x):: - $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade --dry-run - $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade + $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade \ + --target-version=<old-ver> --dry-run + $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade \ + --target-version=<old-ver> You may want to copy all the messages about features that have been removed during the downgrade, in case you want to restore them when diff --git a/test/py/cfgupgrade_unittest.py b/test/py/cfgupgrade_unittest.py index 52a9299..09eb843 100755 --- a/test/py/cfgupgrade_unittest.py +++ b/test/py/cfgupgrade_unittest.py @@ -38,7 +38,7 @@ import testutils def _RunUpgrade(path, dry_run, no_verify, ignore_hostname=True, - downgrade=False): + downgrade=False, target_version=None): cmd = [sys.executable, "%s/tools/cfgupgrade" % testutils.GetSourceDir(), "--debug", "--force", "--path=%s" % path, "--confdir=%s" % path] @@ -50,6 +50,8 @@ def _RunUpgrade(path, dry_run, no_verify, ignore_hostname=True, cmd.append("--no-verify") if downgrade: cmd.append("--downgrade") + if target_version: + cmd.append("--target-version=%s" % target_version) result = utils.RunCmd(cmd, cwd=os.getcwd()) if result.failed: @@ -143,13 +145,14 @@ class TestCfgupgrade(unittest.TestCase): utils.WriteFile(self.config_path, data=serializer.DumpJson({})) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, False, True) - def _TestUpgradeFromFile(self, filename, dry_run): + def _TestUpgradeFromFile(self, filename, dry_run, target_version=None): cfg = self._LoadTestDataConfig(filename) - self._TestUpgradeFromData(cfg, dry_run) + self._TestUpgradeFromData(cfg, dry_run, target_version=target_version) def _TestSimpleUpgrade(self, from_version, dry_run, file_storage_dir=None, - shared_file_storage_dir=None): + shared_file_storage_dir=None, + target_version=None): cluster = {} if file_storage_dir: @@ -163,9 +166,9 @@ class TestCfgupgrade(unittest.TestCase): "instances": {}, "nodegroups": {}, } - self._TestUpgradeFromData(cfg, dry_run) + self._TestUpgradeFromData(cfg, dry_run, target_version=target_version) - def _TestUpgradeFromData(self, cfg, dry_run): + def _TestUpgradeFromData(self, cfg, dry_run, target_version=None): assert "version" in cfg from_version = cfg["version"] self._CreateValidConfigDir() @@ -175,13 +178,17 @@ class TestCfgupgrade(unittest.TestCase): self.assertFalse(os.path.isfile(self.confd_hmac_path)) self.assertFalse(os.path.isfile(self.cds_path)) - _RunUpgrade(self.tmpdir, dry_run, True) + _RunUpgrade(self.tmpdir, dry_run, True, target_version=target_version) if dry_run: expversion = from_version checkfn = operator.not_ else: - expversion = constants.CONFIG_VERSION + if target_version is not None: + parts = target_version.split(".") + expversion = constants.BuildVersion(int(parts[0]), int(parts[1]), 0) + else: + expversion = constants.CONFIG_VERSION checkfn = operator.truth self.assert_(checkfn(os.path.isfile(self.rapi_cert_path))) @@ -362,16 +369,20 @@ class TestCfgupgrade(unittest.TestCase): def testUpgradeFrom_2_7(self): self._TestSimpleUpgrade(constants.BuildVersion(2, 7, 0), False) + def testUpgradeFrom_2_8(self): + self._TestSimpleUpgrade(constants.BuildVersion(2, 8, 0), False) + def testUpgradeFullConfigFrom_2_7(self): self._TestUpgradeFromFile("cluster_config_2.7.json", False) def testUpgradeCurrent(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, False) - def _RunDowngradeUpgrade(self): + def _RunDowngradeUpgrade(self, old_version=None, target_version=None): oldconf = self._LoadConfig() - _RunUpgrade(self.tmpdir, False, True, downgrade=True) - _RunUpgrade(self.tmpdir, False, True) + _RunUpgrade(self.tmpdir, False, True, downgrade=True, + target_version=old_version) + _RunUpgrade(self.tmpdir, False, True, target_version=target_version) newconf = self._LoadConfig() self.assertEqual(oldconf, newconf) @@ -386,7 +397,8 @@ class TestCfgupgrade(unittest.TestCase): # don't override instance specs, so we need to use an ad-hoc configuration. oldconfname = "cluster_config_downgraded_2.7.json" self._TestUpgradeFromFile(oldconfname, False) - _RunUpgrade(self.tmpdir, False, True, downgrade=True) + _RunUpgrade(self.tmpdir, False, True, downgrade=True, + target_version="2.7") oldconf = self._LoadTestDataConfig(oldconfname) newconf = self._LoadConfig() self.assertEqual(oldconf, newconf) @@ -396,11 +408,19 @@ class TestCfgupgrade(unittest.TestCase): self._TestUpgradeFromFile("cluster_config_2.7.json", False) self._RunDowngradeUpgrade() - def _RunDowngradeTwice(self): + def testDowngradeFullConfigBackwardFrom_2_9(self): + """Test for upgrade + downgrade + upgrade combination.""" + self._TestUpgradeFromFile("cluster_config_2.7.json", False, + target_version="2.9") + self._RunDowngradeUpgrade(old_version="2.8", target_version="2.9") + + def _RunDowngradeTwice(self, target_version=None): """Make sure that downgrade is idempotent.""" - _RunUpgrade(self.tmpdir, False, True, downgrade=True) + _RunUpgrade(self.tmpdir, False, True, downgrade=True, + target_version=target_version) oldconf = self._LoadConfig() - _RunUpgrade(self.tmpdir, False, True, downgrade=True) + _RunUpgrade(self.tmpdir, False, True, downgrade=True, + target_version=target_version) newconf = self._LoadConfig() self.assertEqual(oldconf, newconf) @@ -433,6 +453,12 @@ class TestCfgupgrade(unittest.TestCase): def testUpgradeDryRunFrom_2_6(self): self._TestSimpleUpgrade(constants.BuildVersion(2, 6, 0), True) + def testUpgradeDryRunFrom_2_7(self): + self._TestSimpleUpgrade(constants.BuildVersion(2, 7, 0), True) + + def testUpgradeDryRunFrom_2_8(self): + self._TestSimpleUpgrade(constants.BuildVersion(2, 8, 0), True) + def testUpgradeCurrentDryRun(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, True) diff --git a/tools/cfgupgrade b/tools/cfgupgrade index 0526b1a..5ce57f2 100755 --- a/tools/cfgupgrade +++ b/tools/cfgupgrade @@ -49,21 +49,16 @@ options = None args = None -#: Target major version we will upgrade to -TARGET_MAJOR = 2 -#: Target minor version we will upgrade to -TARGET_MINOR = 7 -#: Target major version for downgrade -DOWNGRADE_MAJOR = 2 -#: Target minor version for downgrade -DOWNGRADE_MINOR = 7 - - class Error(Exception): """Generic exception""" pass +def SplitVersion(version): + parts = version.split(".") + return int(parts[0]), int(parts[1]) + + def SetupLogging(): """Configures the logging module. @@ -126,13 +121,13 @@ def UpgradeIPolicy(ipolicy, default_ipolicy, isgroup): _FillIPolicySpecs(default_ipolicy, ipolicy) -def UpgradeNetworks(config_data): +def UpgradeNetworksTo27(config_data): networks = config_data.get("networks", None) if not networks: config_data["networks"] = {} -def UpgradeCluster(config_data): +def UpgradeClusterTo27(config_data): cluster = config_data.get("cluster", None) if cluster is None: raise Error("Cannot find cluster") @@ -141,7 +136,7 @@ def UpgradeCluster(config_data): UpgradeIPolicy(ipolicy, constants.IPOLICY_DEFAULTS, False) -def UpgradeGroups(config_data): +def UpgradeGroupsTo27(config_data): cl_ipolicy = config_data["cluster"].get("ipolicy") for group in config_data["nodegroups"].values(): networks = group.get("networks", None) @@ -155,7 +150,7 @@ def UpgradeGroups(config_data): UpgradeIPolicy(ipolicy, cl_ipolicy, True) -def UpgradeInstances(config_data): +def UpgradeInstancesTo27(config_data): network2uuid = dict((n["name"], n["uuid"]) for n in config_data["networks"].values()) if "instances" not in config_data: @@ -184,7 +179,7 @@ def UpgradeInstances(config_data): dobj["iv_name"] = expected -def UpgradeRapiUsers(): +def UpgradeRapiUsersTo27(): if (os.path.isfile(options.RAPI_USERS_FILE_PRE24) and not os.path.islink(options.RAPI_USERS_FILE_PRE24)): if os.path.exists(options.RAPI_USERS_FILE): @@ -207,7 +202,7 @@ def UpgradeRapiUsers(): os.symlink(options.RAPI_USERS_FILE, options.RAPI_USERS_FILE_PRE24) -def UpgradeWatcher(): +def UpgradeWatcherTo27(): # Remove old watcher state file if it exists if os.path.exists(options.WATCHER_STATEFILE): logging.info("Removing watcher state file %s", options.WATCHER_STATEFILE) @@ -215,7 +210,7 @@ def UpgradeWatcher(): utils.RemoveFile(options.WATCHER_STATEFILE) -def UpgradeFileStoragePaths(config_data): +def UpgradeFileStoragePathsTo27(config_data): # Write file storage paths if not os.path.exists(options.FILE_STORAGE_PATHS_FILE): cluster = config_data["cluster"] @@ -248,19 +243,29 @@ def UpgradeFileStoragePaths(config_data): backup=True) -def UpgradeAll(config_data): - config_data["version"] = constants.BuildVersion(TARGET_MAJOR, - TARGET_MINOR, 0) - UpgradeRapiUsers() - UpgradeWatcher() - UpgradeFileStoragePaths(config_data) - UpgradeNetworks(config_data) - UpgradeCluster(config_data) - UpgradeGroups(config_data) - UpgradeInstances(config_data) +def UpgradeTo27(config_data): + UpgradeRapiUsersTo27() + UpgradeWatcherTo27() + UpgradeFileStoragePathsTo27(config_data) + UpgradeNetworksTo27(config_data) + UpgradeClusterTo27(config_data) + UpgradeGroupsTo27(config_data) + UpgradeInstancesTo27(config_data) -def DowngradeIPolicy(ipolicy, owner): +def UpgradeInstancesTo29(config_data): + if "instances" not in config_data: + raise Error("Can't find the 'instances' key in the configuration!") + + for _, iobj in config_data["instances"].items(): + iobj["disks_active"] = iobj["admin_state"] == constants.ADMINST_UP + + +def UpgradeTo29(config_data): + UpgradeInstancesTo29(config_data) + + +def DowngradeIPolicyTo27(ipolicy, owner): # Downgrade IPolicy to 2.7 (stable) minmax_keys = ["min", "max"] specs_is_split = any((k in ipolicy) for k in minmax_keys) @@ -283,14 +288,14 @@ def DowngradeIPolicy(ipolicy, owner): ipolicy["std"] = {} -def DowngradeGroups(config_data): +def DowngradeGroupsTo26(config_data): for group in config_data["nodegroups"].values(): ipolicy = group.get("ipolicy", None) if ipolicy is not None: - DowngradeIPolicy(ipolicy, "group \"%s\"" % group.get("name")) + DowngradeIPolicyTo27(ipolicy, "group \"%s\"" % group.get("name")) -def DowngradeEnabledTemplates(cluster): +def DowngradeEnabledTemplatesTo26(cluster): # Remove enabled disk templates to downgrade to 2.7 edt_key = "enabled_disk_templates" if edt_key in cluster: @@ -299,21 +304,31 @@ def DowngradeEnabledTemplates(cluster): del cluster[edt_key] -def DowngradeCluster(config_data): +def DowngradeClusterTo26(config_data): cluster = config_data.get("cluster", None) if cluster is None: raise Error("Cannot find cluster") - DowngradeEnabledTemplates(cluster) + DowngradeEnabledTemplatesTo26(cluster) ipolicy = cluster.get("ipolicy", None) if ipolicy: - DowngradeIPolicy(ipolicy, "cluster") + DowngradeIPolicyTo27(ipolicy, "cluster") -def DowngradeAll(config_data): - # Any code specific to a particular version should be labeled that way, so - # it can be removed when updating to the next version. - DowngradeCluster(config_data) - DowngradeGroups(config_data) +def DowngradeTo26(config_data): + DowngradeClusterTo26(config_data) + DowngradeGroupsTo26(config_data) + + +def DowngradeInstancesTo28(config_data): + if "instances" not in config_data: + raise Error("Can't find the 'instances' key in the configuration!") + + for _, iobj in config_data["instances"].items(): + del iobj["disks_active"] + + +def DowngradeTo28(config_data): + DowngradeInstancesTo28(config_data) def main(): @@ -347,6 +362,12 @@ def main(): parser.add_option("--downgrade", help="Downgrade to the previous stable version", action="store_true", dest="downgrade", default=False) + parser.add_option("--target-version", + help="Target version to upgrade/downgrade to", + dest="target_version", + default=("%d.%d" % + (constants.CONFIG_MAJOR, + constants.CONFIG_MINOR))) (options, args) = parser.parse_args() # We need to keep filenames locally because they might be renamed between @@ -365,6 +386,8 @@ def main(): options.SSCONF_MASTER_NODE = options.data_dir + "/ssconf_master_node" options.WATCHER_STATEFILE = options.data_dir + "/watcher.data" options.FILE_STORAGE_PATHS_FILE = options.conf_dir + "/file-storage-paths" + options.TARGET_MAJOR, options.TARGET_MINOR = \ + SplitVersion(options.target_version) SetupLogging() @@ -387,7 +410,7 @@ def main(): " upgrade notes (available in the UPGRADE file and included" " in other documentation formats) to understand what they" " are. Continue with *DOWNGRADING* the configuration?" % - (DOWNGRADE_MAJOR, DOWNGRADE_MINOR)) + (options.TARGET_MAJOR, options.TARGET_MINOR)) else: usertext = ("Please make sure you have read the upgrade notes for" " Ganeti %s (available in the UPGRADE file and included" @@ -423,27 +446,38 @@ def main(): raise Error("Inconsistent configuration: found config_version in" " configuration file") + if config_major != 2 or options.TARGET_MAJOR != 2: + raise Error("Upgrade/downgrade supported only for 2.x series!") + + if config_revision != 0: + logging.warning("Config revision is %s, not 0", config_revision) + # Downgrade to the previous stable version if options.downgrade: - if config_major != TARGET_MAJOR or config_minor != TARGET_MINOR: - raise Error("Downgrade supported only from the latest version (%s.%s)," - " found %s (%s.%s.%s) instead" % - (TARGET_MAJOR, TARGET_MINOR, config_version, config_major, - config_minor, config_revision)) - DowngradeAll(config_data) - - # Upgrade from 2.{0..7} to 2.7 - elif config_major == 2 and config_minor in range(0, 8): - if config_revision != 0: - logging.warning("Config revision is %s, not 0", config_revision) - UpgradeAll(config_data) - - elif config_major == TARGET_MAJOR and config_minor == TARGET_MINOR: - logging.info("No changes necessary") + if config_minor >= 9 and options.TARGET_MINOR <= 8: + DowngradeTo28(config_data) + if config_minor >= 8 and options.TARGET_MINOR <= 7: + DowngradeTo26(config_data) else: - raise Error("Configuration version %d.%d.%d not supported by this tool" % - (config_major, config_minor, config_revision)) + if config_minor in range(0, 7) and options.TARGET_MINOR >= 7: + # Upgrade from 2.{0..6} to 2.7 + UpgradeTo27(config_data) + + if config_minor in range(0, 9) and options.TARGET_MINOR >= 9: + # Upgrade from 2.{0..8} to 2.9 + UpgradeTo29(config_data) + + if config_major == options.TARGET_MAJOR and \ + config_minor == options.TARGET_MINOR: + logging.info("No changes necessary") + + if config_major != 2 or config_minor not in range(0, 10): + raise Error("Configuration version %d.%d.%d not supported by this tool" % + (config_major, config_minor, config_revision)) + + config_data["version"] = constants.BuildVersion(options.TARGET_MAJOR, + options.TARGET_MINOR, 0) try: logging.info("Writing configuration file to %s", options.CONFIG_DATA_PATH) @@ -486,12 +520,12 @@ def main(): logging.info("File loaded successfully after upgrading") del cfg + out_ver = "%s.%s" % (options.TARGET_MAJOR, options.TARGET_MINOR) if options.downgrade: action = "downgraded" - out_ver = "%s.%s" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR) else: action = "upgraded" - out_ver = constants.RELEASE_VERSION + if all_ok: cli.ToStderr("Configuration successfully %s to version %s.", action, out_ver) -- 1.8.2.1
