commit d882cffd5ec46fea6dc5e7a204b92cd32a144dda
Merge: 5f58149 eaaeb76
Author: Klaus Aehlig <[email protected]>
Date: Fri Apr 24 13:53:15 2015 +0200
Merge branch 'stable-2.13' into stable-2.14
* stable-2.13
Fix sample 2.12 configuration
NodeSshRemoveKey: Add retries for updating the target node
SSH file manager: properly name assertion function
Unittests: Simplify generating a master candidate node
RemoveNodeSshKey: use retries when updating other nodes
AddNodeSshKey: retries for all non-master nodes
AddNodeSshKey: retry when target node not reachable
Remove obsolete constant SSHS_RENAME
Simplify testdata setup and teardown
Consider offline nodes when removing SSH keys
Consider offline nodes in NodeSshKeyAdd
Use SSH file manager for unittests removing keys
Use SSH file manager in key adding unit tests
Introduce (testutils) SSH file manager
Only delete old node keys in one-key-setup
Add debug comments to RenewCrypto
Fix renewing master node's SSH key
Create CHROOT directory before copy COMP_FILENAME
* stable-2.12
When assigning UUIDs to disks, do so recursively
Fix sample 2.11 configuration
Include hypervisor parameters in SSConf
Add SSConf keys for hypervisor parameters
Use Hypervisor as the key in ClusterHvParams
Re-remove final config update in renew-crypto
Fix string formatting in private object representation
Fix the computation of the list of reserved IP addresses
Increase number of retries for daemon RPCs
* stable-2.11
Update configure file to version 2.11.7
Update NEWS file for 2.11.7 release
Add logging to RenewCrypto
Fix format string for gnt-network info
Replace textwrapper.wrap by a custom version for networks
Add SSL improvements to NEWS file
* stable-2.10
Update tag limitations
Fix typos in doc/design-storagetypes.rst
Make getFQDN prefer cluster protocol family
Add version of getFQDN accepting preferences
Make getFQDN honor vcluster
Conflicts:
Makefile.am
lib/cmdlib/cluster/verify.py
src/Ganeti/Config.hs
tools/cfgupgrade
Resolved by manually following code moves.
Signed-off-by: Klaus Aehlig <[email protected]>
diff --cc Makefile.am
index 05a9b81,518eb2b..2aa6865
--- a/Makefile.am
+++ b/Makefile.am
@@@ -1925,9 -1841,9 +1925,10 @@@ python_tests =
python_test_support = \
test/py/__init__.py \
test/py/lockperf.py \
- test/py/testutils.py \
+ test/py/testutils_ssh.py \
test/py/mocks.py \
+ test/py/testutils/__init__.py \
+ test/py/testutils/config_mock.py \
test/py/cmdlib/__init__.py \
test/py/cmdlib/testsupport/__init__.py \
test/py/cmdlib/testsupport/cmdlib_testcase.py \
diff --cc lib/tools/cfgupgrade.py
index 5a52dd7,0000000..b31d22b
mode 100644,000000..100644
--- a/lib/tools/cfgupgrade.py
+++ b/lib/tools/cfgupgrade.py
@@@ -1,841 -1,0 +1,847 @@@
+#
+#
+
+# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Library of the tools/cfgupgrade utility.
+
+This code handles only the types supported by simplejson. As an
+example, 'set' is a 'list'.
+
+"""
+
+import copy
+import os
+import os.path
+import sys
+import logging
+import optparse
+import time
+import functools
+from cStringIO import StringIO
+
+from ganeti import cli
+from ganeti import constants
+from ganeti import serializer
+from ganeti import utils
+from ganeti import bootstrap
+from ganeti import config
+from ganeti import pathutils
+from ganeti import netutils
+
+from ganeti.utils import version
+
+
+#: Target major version we will upgrade to
+TARGET_MAJOR = 2
+#: Target minor version we will upgrade to
+TARGET_MINOR = 14
+#: Target major version for downgrade
+DOWNGRADE_MAJOR = 2
+#: Target minor version for downgrade
+DOWNGRADE_MINOR = 13
+
+# map of legacy device types
+# (mapping differing old LD_* constants to new DT_* constants)
+DEV_TYPE_OLD_NEW = {"lvm": constants.DT_PLAIN, "drbd8": constants.DT_DRBD8}
+# (mapping differing new DT_* constants to old LD_* constants)
+DEV_TYPE_NEW_OLD = dict((v, k) for k, v in DEV_TYPE_OLD_NEW.items())
+
+
+class Error(Exception):
+ """Generic exception"""
+ pass
+
+
+def ParseOptions(args=None):
+ parser = optparse.OptionParser(usage="%prog [--debug|--verbose] [--force]")
+ parser.add_option("--dry-run", dest="dry_run",
+ action="store_true",
+ help="Try to do the conversion, but don't write"
+ " output file")
+ parser.add_option(cli.FORCE_OPT)
+ parser.add_option(cli.DEBUG_OPT)
+ parser.add_option(cli.VERBOSE_OPT)
+ parser.add_option("--ignore-hostname", dest="ignore_hostname",
+ action="store_true", default=False,
+ help="Don't abort if hostname doesn't match")
+ parser.add_option("--path", help="Convert configuration in this"
+ " directory instead of '%s'" % pathutils.DATA_DIR,
+ default=pathutils.DATA_DIR, dest="data_dir")
+ parser.add_option("--confdir",
+ help=("Use this directory instead of '%s'" %
+ pathutils.CONF_DIR),
+ default=pathutils.CONF_DIR, dest="conf_dir")
+ parser.add_option("--no-verify",
+ help="Do not verify configuration after upgrade",
+ action="store_true", dest="no_verify", default=False)
+ parser.add_option("--downgrade",
+ help="Downgrade to the previous stable version",
+ action="store_true", dest="downgrade", default=False)
+ return parser.parse_args(args=args)
+
+
+def OrFail(description=None):
+ """Make failure non-fatal and improve reporting."""
+ def wrapper(f):
+ @functools.wraps(f)
+ def wrapped(self):
+ safety = copy.deepcopy(self.config_data)
+ try:
+ f(self)
+ except BaseException, e:
+ msg = "%s failed:\n%s" % (description or f.func_name, e)
+ logging.error(msg)
+ self.config_data = safety
+ self.errors.append(msg)
+ return wrapped
+ return wrapper
+
+
+class CfgUpgrade(object):
+ def __init__(self, opts, args):
+ self.opts = opts
+ self.args = args
+ self.errors = []
+
+ def Run(self):
+ """Main program.
+
+ """
+ self._ComposePaths()
+
+ self.SetupLogging()
+
+ # Option checking
+ if self.args:
+ raise Error("No arguments expected")
+ if self.opts.downgrade and not self.opts.no_verify:
+ self.opts.no_verify = True
+
+ # Check master name
+ if not (self.CheckHostname(self.opts.SSCONF_MASTER_NODE) or
+ self.opts.ignore_hostname):
+ logging.error("Aborting due to hostname mismatch")
+ sys.exit(constants.EXIT_FAILURE)
+
+ self._AskUser()
+
+ # Check whether it's a Ganeti configuration directory
+ if not (os.path.isfile(self.opts.CONFIG_DATA_PATH) and
+ os.path.isfile(self.opts.SERVER_PEM_PATH) and
+ os.path.isfile(self.opts.KNOWN_HOSTS_PATH)):
+ raise Error(("%s does not seem to be a Ganeti configuration"
+ " directory") % self.opts.data_dir)
+
+ if not os.path.isdir(self.opts.conf_dir):
+ raise Error("Not a directory: %s" % self.opts.conf_dir)
+
+ self.config_data = serializer.LoadJson(utils.ReadFile(
+ self.opts.CONFIG_DATA_PATH))
+
+ try:
+ config_version = self.config_data["version"]
+ except KeyError:
+ raise Error("Unable to determine configuration version")
+
+ (config_major, config_minor, config_revision) = \
+ version.SplitVersion(config_version)
+
+ logging.info("Found configuration version %s (%d.%d.%d)",
+ config_version, config_major, config_minor, config_revision)
+
+ if "config_version" in self.config_data["cluster"]:
+ raise Error("Inconsistent configuration: found config_version in"
+ " configuration file")
+
+ # Downgrade to the previous stable version
+ if self.opts.downgrade:
+ self._Downgrade(config_major, config_minor, config_version,
+ config_revision)
+
+ # Upgrade from 2.{0..13} to 2.14
+ elif config_major == 2 and config_minor in range(0, 14):
+ if config_revision != 0:
+ logging.warning("Config revision is %s, not 0", config_revision)
+ if not self.UpgradeAll():
+ raise Error("Upgrade failed:\n%s", '\n'.join(self.errors))
+
+ elif config_major == TARGET_MAJOR and config_minor == TARGET_MINOR:
+ logging.info("No changes necessary")
+
+ else:
+ raise Error("Configuration version %d.%d.%d not supported by this tool"
%
+ (config_major, config_minor, config_revision))
+
+ try:
+ logging.info("Writing configuration file to %s",
+ self.opts.CONFIG_DATA_PATH)
+ utils.WriteFile(file_name=self.opts.CONFIG_DATA_PATH,
+ data=serializer.DumpJson(self.config_data),
+ mode=0600,
+ dry_run=self.opts.dry_run,
+ backup=True)
+
+ if not self.opts.dry_run:
+ bootstrap.GenerateClusterCrypto(
+ False, False, False, False, False,
+ nodecert_file=self.opts.SERVER_PEM_PATH,
+ rapicert_file=self.opts.RAPI_CERT_FILE,
+ spicecert_file=self.opts.SPICE_CERT_FILE,
+ spicecacert_file=self.opts.SPICE_CACERT_FILE,
+ hmackey_file=self.opts.CONFD_HMAC_KEY,
+ cds_file=self.opts.CDS_FILE)
+
+ except Exception:
+ logging.critical("Writing configuration failed. It is probably in an"
+ " inconsistent state and needs manual intervention.")
+ raise
+
+ self._TestLoadingConfigFile()
+
+ def SetupLogging(self):
+ """Configures the logging module.
+
+ """
+ formatter = logging.Formatter("%(asctime)s: %(message)s")
+
+ stderr_handler = logging.StreamHandler()
+ stderr_handler.setFormatter(formatter)
+ if self.opts.debug:
+ stderr_handler.setLevel(logging.NOTSET)
+ elif self.opts.verbose:
+ stderr_handler.setLevel(logging.INFO)
+ else:
+ stderr_handler.setLevel(logging.WARNING)
+
+ root_logger = logging.getLogger("")
+ root_logger.setLevel(logging.NOTSET)
+ root_logger.addHandler(stderr_handler)
+
+ @staticmethod
+ def CheckHostname(path):
+ """Ensures hostname matches ssconf value.
+
+ @param path: Path to ssconf file
+
+ """
+ ssconf_master_node = utils.ReadOneLineFile(path)
+ hostname = netutils.GetHostname().name
+
+ if ssconf_master_node == hostname:
+ return True
+
+ logging.warning("Warning: ssconf says master node is '%s', but this"
+ " machine's name is '%s'; this tool must be run on"
+ " the master node", ssconf_master_node, hostname)
+ return False
+
+ @staticmethod
+ def _FillIPolicySpecs(default_ipolicy, ipolicy):
+ if "minmax" in ipolicy:
+ for (key, spec) in ipolicy["minmax"][0].items():
+ for (par, val) in default_ipolicy["minmax"][0][key].items():
+ if par not in spec:
+ spec[par] = val
+
+ def UpgradeIPolicy(self, ipolicy, default_ipolicy, isgroup):
+ minmax_keys = ["min", "max"]
+ if any((k in ipolicy) for k in minmax_keys):
+ minmax = {}
+ for key in minmax_keys:
+ if key in ipolicy:
+ if ipolicy[key]:
+ minmax[key] = ipolicy[key]
+ del ipolicy[key]
+ if minmax:
+ ipolicy["minmax"] = [minmax]
+ if isgroup and "std" in ipolicy:
+ del ipolicy["std"]
+ self._FillIPolicySpecs(default_ipolicy, ipolicy)
+
+ @OrFail("Setting networks")
+ def UpgradeNetworks(self):
+ assert isinstance(self.config_data, dict)
+ # pylint can't infer config_data type
+ # pylint: disable=E1103
+ networks = self.config_data.get("networks", None)
+ if not networks:
+ self.config_data["networks"] = {}
+
+ @OrFail("Upgrading cluster")
+ def UpgradeCluster(self):
+ assert isinstance(self.config_data, dict)
+ # pylint can't infer config_data type
+ # pylint: disable=E1103
+ cluster = self.config_data.get("cluster", None)
+ if cluster is None:
+ raise Error("Cannot find cluster")
+ ipolicy = cluster.setdefault("ipolicy", None)
+ if ipolicy:
+ self.UpgradeIPolicy(ipolicy, constants.IPOLICY_DEFAULTS, False)
+ ial_params = cluster.get("default_iallocator_params", None)
+ if not ial_params:
+ cluster["default_iallocator_params"] = {}
+ if not "candidate_certs" in cluster:
+ cluster["candidate_certs"] = {}
+ cluster["instance_communication_network"] = \
+ cluster.get("instance_communication_network", "")
+ cluster["install_image"] = \
+ cluster.get("install_image", "")
+ cluster["zeroing_image"] = \
+ cluster.get("zeroing_image", "")
+ cluster["compression_tools"] = \
+ cluster.get("compression_tools", constants.IEC_DEFAULT_TOOLS)
+ if "enabled_user_shutdown" not in cluster:
+ cluster["enabled_user_shutdown"] = False
+ cluster["data_collectors"] = cluster.get("data_collectors", {})
+ for name in constants.DATA_COLLECTOR_NAMES:
+ cluster["data_collectors"][name] = \
+ cluster["data_collectors"].get(
+ name, dict(active=True,
+ interval=constants.MOND_TIME_INTERVAL * 1e6))
+
+ @OrFail("Upgrading groups")
+ def UpgradeGroups(self):
+ cl_ipolicy = self.config_data["cluster"].get("ipolicy")
+ for group in self.config_data["nodegroups"].values():
+ networks = group.get("networks", None)
+ if not networks:
+ group["networks"] = {}
+ ipolicy = group.get("ipolicy", None)
+ if ipolicy:
+ if cl_ipolicy is None:
+ raise Error("A group defines an instance policy but there is no"
+ " instance policy at cluster level")
+ self.UpgradeIPolicy(ipolicy, cl_ipolicy, True)
+
+ def GetExclusiveStorageValue(self):
+ """Return a conservative value of the exclusive_storage flag.
+
+ Return C{True} if the cluster or at least a nodegroup have the flag set.
+
+ """
+ ret = False
+ cluster = self.config_data["cluster"]
+ ndparams = cluster.get("ndparams")
+ if ndparams is not None and ndparams.get("exclusive_storage"):
+ ret = True
+ for group in self.config_data["nodegroups"].values():
+ ndparams = group.get("ndparams")
+ if ndparams is not None and ndparams.get("exclusive_storage"):
+ ret = True
+ return ret
+
+ def RemovePhysicalId(self, disk):
+ if "children" in disk:
+ for d in disk["children"]:
+ self.RemovePhysicalId(d)
+ if "physical_id" in disk:
+ del disk["physical_id"]
+
+ def ChangeDiskDevType(self, disk, dev_type_map):
+ """Replaces disk's dev_type attributes according to the given map.
+
+ This can be used for both, up or downgrading the disks.
+ """
+ if disk["dev_type"] in dev_type_map:
+ disk["dev_type"] = dev_type_map[disk["dev_type"]]
+ if "children" in disk:
+ for child in disk["children"]:
+ self.ChangeDiskDevType(child, dev_type_map)
+
+ def UpgradeDiskDevType(self, disk):
+ """Upgrades the disks' device type."""
+ self.ChangeDiskDevType(disk, DEV_TYPE_OLD_NEW)
+
+ @staticmethod
+ def _ConvertNicNameToUuid(iobj, network2uuid):
+ for nic in iobj["nics"]:
+ name = nic.get("network", None)
+ if name:
+ uuid = network2uuid.get(name, None)
+ if uuid:
+ print("NIC with network name %s found."
+ " Substituting with uuid %s." % (name, uuid))
+ nic["network"] = uuid
+
++ def AssignUuid(disk):
++ if not "uuid" in disk:
++ disk["uuid"] = utils.io.NewUUID()
++ if "children" in disk:
++ for d in disk["children"]:
++ AssignUuid(d)
++
+ def _ConvertDiskAndCheckMissingSpindles(self, iobj, instance):
+ missing_spindles = False
+ if "disks" not in iobj:
+ raise Error("Instance '%s' doesn't have a disks entry?!" % instance)
+ disks = iobj["disks"]
+ if not all(isinstance(d, str) for d in disks):
+ # Disks are not top level citizens
+ for idx, dobj in enumerate(disks):
+ self.RemovePhysicalId(dobj)
+
+ expected = "disk/%s" % idx
+ current = dobj.get("iv_name", "")
+ if current != expected:
+ logging.warning("Updating iv_name for instance %s/disk %s"
+ " from '%s' to '%s'",
+ instance, idx, current, expected)
+ dobj["iv_name"] = expected
+
+ if "dev_type" in dobj:
+ self.UpgradeDiskDevType(dobj)
+
+ if not "spindles" in dobj:
+ missing_spindles = True
+
- if not "uuid" in dobj:
- dobj["uuid"] = utils.io.NewUUID()
++ AssignUuid(dobj)
+ return missing_spindles
+
+ @OrFail("Upgrading instance with spindles")
+ def UpgradeInstances(self):
+ """Upgrades the instances' configuration."""
+
+ network2uuid = dict((n["name"], n["uuid"])
+ for n in self.config_data["networks"].values())
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' key in the configuration!")
+
+ missing_spindles = False
+ for instance, iobj in self.config_data["instances"].items():
+ self._ConvertNicNameToUuid(iobj, network2uuid)
+ if self._ConvertDiskAndCheckMissingSpindles(iobj, instance):
+ missing_spindles = True
+ if "admin_state_source" not in iobj:
+ iobj["admin_state_source"] = constants.ADMIN_SOURCE
+
+ if self.GetExclusiveStorageValue() and missing_spindles:
+ # We cannot be sure that the instances that are missing spindles have
+ # exclusive storage enabled (the check would be more complicated), so we
+ # give a noncommittal message
+ logging.warning("Some instance disks could be needing to update the"
+ " spindles parameter; you can check by running"
+ " 'gnt-cluster verify', and fix any problem with"
+ " 'gnt-cluster repair-disk-sizes'")
+
+ def UpgradeRapiUsers(self):
+ if (os.path.isfile(self.opts.RAPI_USERS_FILE_PRE24) and
+ not os.path.islink(self.opts.RAPI_USERS_FILE_PRE24)):
+ if os.path.exists(self.opts.RAPI_USERS_FILE):
+ raise Error("Found pre-2.4 RAPI users file at %s, but another file"
+ " already exists at %s" %
+ (self.opts.RAPI_USERS_FILE_PRE24,
+ self.opts.RAPI_USERS_FILE))
+ logging.info("Found pre-2.4 RAPI users file at %s, renaming to %s",
+ self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE)
+ if not self.opts.dry_run:
+ utils.RenameFile(self.opts.RAPI_USERS_FILE_PRE24,
+ self.opts.RAPI_USERS_FILE, mkdir=True,
mkdir_mode=0750)
+
+ # Create a symlink for RAPI users file
+ if (not (os.path.islink(self.opts.RAPI_USERS_FILE_PRE24) or
+ os.path.isfile(self.opts.RAPI_USERS_FILE_PRE24)) and
+ os.path.isfile(self.opts.RAPI_USERS_FILE)):
+ logging.info("Creating symlink from %s to %s",
+ self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE)
+ if not self.opts.dry_run:
+ os.symlink(self.opts.RAPI_USERS_FILE, self.opts.RAPI_USERS_FILE_PRE24)
+
+ def UpgradeWatcher(self):
+ # Remove old watcher state file if it exists
+ if os.path.exists(self.opts.WATCHER_STATEFILE):
+ logging.info("Removing watcher state file %s",
+ self.opts.WATCHER_STATEFILE)
+ if not self.opts.dry_run:
+ utils.RemoveFile(self.opts.WATCHER_STATEFILE)
+
+ @OrFail("Upgrading file storage paths")
+ def UpgradeFileStoragePaths(self):
+ # Write file storage paths
+ if not os.path.exists(self.opts.FILE_STORAGE_PATHS_FILE):
+ cluster = self.config_data["cluster"]
+ file_storage_dir = cluster.get("file_storage_dir")
+ shared_file_storage_dir = cluster.get("shared_file_storage_dir")
+ del cluster
+
+ logging.info("Ganeti 2.7 and later only allow whitelisted directories"
+ " for file storage; writing existing configuration values"
+ " into '%s'",
+ self.opts.FILE_STORAGE_PATHS_FILE)
+
+ if file_storage_dir:
+ logging.info("File storage directory: %s", file_storage_dir)
+ if shared_file_storage_dir:
+ logging.info("Shared file storage directory: %s",
+ shared_file_storage_dir)
+
+ buf = StringIO()
+ buf.write("# List automatically generated from configuration by\n")
+ buf.write("# cfgupgrade at %s\n" % time.asctime())
+ if file_storage_dir:
+ buf.write("%s\n" % file_storage_dir)
+ if shared_file_storage_dir:
+ buf.write("%s\n" % shared_file_storage_dir)
+ utils.WriteFile(file_name=self.opts.FILE_STORAGE_PATHS_FILE,
+ data=buf.getvalue(),
+ mode=0600,
+ dry_run=self.opts.dry_run,
+ backup=True)
+
+ @staticmethod
+ def GetNewNodeIndex(nodes_by_old_key, old_key, new_key_field):
+ if old_key not in nodes_by_old_key:
+ logging.warning("Can't find node '%s' in configuration, "
+ "assuming that it's already up-to-date", old_key)
+ return old_key
+ return nodes_by_old_key[old_key][new_key_field]
+
+ def ChangeNodeIndices(self, config_data, old_key_field, new_key_field):
+ def ChangeDiskNodeIndices(disk):
+ # Note: 'drbd8' is a legacy device type from pre 2.9 and needs to be
+ # considered when up/downgrading from/to any versions touching 2.9 on
the
+ # way.
+ drbd_disk_types = set(["drbd8"]) | constants.DTS_DRBD
+ if disk["dev_type"] in drbd_disk_types:
+ for i in range(0, 2):
+ disk["logical_id"][i] = self.GetNewNodeIndex(nodes_by_old_key,
+ disk["logical_id"][i],
+ new_key_field)
+ if "children" in disk:
+ for child in disk["children"]:
+ ChangeDiskNodeIndices(child)
+
+ nodes_by_old_key = {}
+ nodes_by_new_key = {}
+ for (_, node) in config_data["nodes"].items():
+ nodes_by_old_key[node[old_key_field]] = node
+ nodes_by_new_key[node[new_key_field]] = node
+
+ config_data["nodes"] = nodes_by_new_key
+
+ cluster = config_data["cluster"]
+ cluster["master_node"] = self.GetNewNodeIndex(nodes_by_old_key,
+ cluster["master_node"],
+ new_key_field)
+
+ for inst in config_data["instances"].values():
+ inst["primary_node"] = self.GetNewNodeIndex(nodes_by_old_key,
+ inst["primary_node"],
+ new_key_field)
+
+ for disk in config_data["disks"].values():
+ ChangeDiskNodeIndices(disk)
+
+ @staticmethod
+ def ChangeInstanceIndices(config_data, old_key_field, new_key_field):
+ insts_by_old_key = {}
+ insts_by_new_key = {}
+ for (_, inst) in config_data["instances"].items():
+ insts_by_old_key[inst[old_key_field]] = inst
+ insts_by_new_key[inst[new_key_field]] = inst
+
+ config_data["instances"] = insts_by_new_key
+
+ @OrFail("Changing node indices")
+ def UpgradeNodeIndices(self):
+ self.ChangeNodeIndices(self.config_data, "name", "uuid")
+
+ @OrFail("Changing instance indices")
+ def UpgradeInstanceIndices(self):
+ self.ChangeInstanceIndices(self.config_data, "name", "uuid")
+
+ @OrFail("Adding filters")
+ def UpgradeFilters(self):
+ # pylint can't infer config_data type
+ # pylint: disable=E1103
+ filters = self.config_data.get("filters", None)
+ if not filters:
+ self.config_data["filters"] = {}
+
+ @OrFail("Set top level disks")
+ def UpgradeTopLevelDisks(self):
+ """Upgrades the disks as config top level citizens."""
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' key in the configuration!")
+
+ if "disks" in self.config_data:
+ # Disks are already top level citizens
+ return
+
+ self.config_data["disks"] = dict()
+ for iobj in self.config_data["instances"].values():
+ disk_uuids = []
+ for disk in iobj["disks"]:
+ duuid = disk["uuid"]
+ disk["serial_no"] = 1
+ disk["ctime"] = disk["mtime"] = iobj["ctime"]
+ self.config_data["disks"][duuid] = disk
+ disk_uuids.append(duuid)
+ iobj["disks"] = disk_uuids
+
+ @OrFail("Removing disk template")
+ def UpgradeDiskTemplate(self):
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' dictionary in the
configuration.")
+ instances = self.config_data["instances"]
+ for inst in instances.values():
+ if "disk_template" in inst:
+ del inst["disk_template"]
+
+ # The following function is based on a method of class Disk with the same
+ # name, but adjusted to work with dicts and sets.
+ def _ComputeAllNodes(self, disk):
+ """Recursively compute nodes given a top device."""
+ nodes = set()
+ if disk["dev_type"] in constants.DTS_DRBD:
+ nodes = set(disk["logical_id"][:2])
+ for child in disk.get("children", []):
+ nodes |= self._ComputeAllNodes(child)
+ return nodes
+
+ def _RecursiveUpdateNodes(self, disk, nodes):
+ disk["nodes"] = nodes
+ for child in disk.get("children", []):
+ self._RecursiveUpdateNodes(child, nodes)
+
+ @OrFail("Upgrading disk nodes")
+ def UpgradeDiskNodes(self):
+ """Specify the nodes from which a disk is accessible in its definition.
+
+ For every disk that is attached to an instance, get the UUIDs of the nodes
+ that it's accessible from. There are three main cases:
+ 1) Internally mirrored disks (DRBD):
+ These disks are accessible from two nodes, so the nodes list will include
+ these. Their children (data, meta) are also accessible from two nodes,
+ therefore they will inherit the nodes of the parent.
+ 2) Externally mirrored disks (Blockdev, Ext, Gluster, RBD, Shared File):
+ These disks should be accessible from any node in the cluster, therefore
the
+ nodes list will be empty.
+ 3) Single-node disks (Plain, File):
+ These disks are accessible from one node only, therefore the nodes list
will
+ consist only of the primary instance node.
+ """
+ disks = self.config_data["disks"]
+ for instance in self.config_data["instances"].itervalues():
+ # Get all disk nodes for an instance
+ instance_node = set([instance["primary_node"]])
+ disk_nodes = set()
+ for disk_uuid in instance["disks"]:
+ disk_nodes |= self._ComputeAllNodes(disks[disk_uuid])
+ all_nodes = list(instance_node | disk_nodes)
+
+ # Populate the `nodes` list field of each disk.
+ for disk_uuid in instance["disks"]:
+ disk = disks[disk_uuid]
+ if "nodes" in disk:
+ # The "nodes" field has already been added for this disk.
+ continue
+
+ if disk["dev_type"] in constants.DTS_INT_MIRROR:
+ self._RecursiveUpdateNodes(disk, all_nodes)
+ elif disk["dev_type"] in (constants.DT_PLAIN, constants.DT_FILE):
+ disk["nodes"] = all_nodes
+ else:
+ disk["nodes"] = []
+
+ def UpgradeAll(self):
+ self.config_data["version"] = version.BuildVersion(TARGET_MAJOR,
+ TARGET_MINOR, 0)
+ self.UpgradeRapiUsers()
+ self.UpgradeWatcher()
+ steps = [self.UpgradeFileStoragePaths,
+ self.UpgradeNetworks,
+ self.UpgradeCluster,
+ self.UpgradeGroups,
+ self.UpgradeInstances,
+ self.UpgradeTopLevelDisks,
+ self.UpgradeNodeIndices,
+ self.UpgradeInstanceIndices,
+ self.UpgradeFilters,
+ self.UpgradeDiskNodes,
+ self.UpgradeDiskTemplate]
+ for s in steps:
+ s()
+ return not self.errors
+
+ # DOWNGRADE ------------------------------------------------------------
+
+ def _RecursiveRemoveNodes(self, disk):
+ if "nodes" in disk:
+ del disk["nodes"]
+ for disk in disk.get("children", []):
+ self._RecursiveRemoveNodes(disk)
+
+ @OrFail("Downgrading disk nodes")
+ def DowngradeDiskNodes(self):
+ if "disks" not in self.config_data:
+ raise Error("Can't find the 'disks' dictionary in the configuration.")
+ for disk in self.config_data["disks"].itervalues():
+ self._RecursiveRemoveNodes(disk)
+
+ @OrFail("Removing forthcoming instances")
+ def DowngradeForthcomingInstances(self):
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' dictionary in the
configuration.")
+ instances = self.config_data["instances"]
+ uuids = instances.keys()
+ for uuid in uuids:
+ if instances[uuid].get("forthcoming"):
+ del instances[uuid]
+
+ @OrFail("Removing forthcoming disks")
+ def DowngradeForthcomingDisks(self):
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' dictionary in the
configuration.")
+ instances = self.config_data["instances"]
+ if "disks" not in self.config_data:
+ raise Error("Can't find the 'disks' dictionary in the configuration.")
+ disks = self.config_data["disks"]
+ uuids = disks.keys()
+ for uuid in uuids:
+ if disks[uuid].get("forthcoming"):
+ del disks[uuid]
+ for inst in instances:
+ if "disk" in inst and uuid in inst["disks"]:
+ inst["disks"].remove(uuid)
+
+ @OrFail("Re-adding disk template")
+ def DowngradeDiskTemplate(self):
+ if "instances" not in self.config_data:
+ raise Error("Can't find the 'instances' dictionary in the
configuration.")
+ instances = self.config_data["instances"]
+ if "disks" not in self.config_data:
+ raise Error("Can't find the 'disks' dictionary in the configuration.")
+ disks = self.config_data["disks"]
+ for inst in instances.values():
+ instance_disks = [disks.get(uuid) for uuid in inst["disks"]]
+ if any(d is None for d in instance_disks):
+ raise Error("Can't find all disks of instance %s in the
configuration."
+ % inst.name)
+ dev_types = set(d["dev_type"] for d in instance_disks)
+ if len(dev_types) > 1:
+ raise Error("Instance %s has mixed disk types: %s" %
+ (inst.name, ', '.join(dev_types)))
+ elif len(dev_types) < 1:
+ inst["disk_template"] = constants.DT_DISKLESS
+ else:
+ inst["disk_template"] = dev_types.pop()
+
+ def DowngradeAll(self):
+ self.config_data["version"] = version.BuildVersion(DOWNGRADE_MAJOR,
+ DOWNGRADE_MINOR, 0)
+ steps = [self.DowngradeForthcomingInstances,
+ self.DowngradeForthcomingDisks,
+ self.DowngradeDiskNodes,
+ self.DowngradeDiskTemplate]
+ for s in steps:
+ s()
+ return not self.errors
+
+ def _ComposePaths(self):
+ # We need to keep filenames locally because they might be renamed between
+ # versions.
+ self.opts.data_dir = os.path.abspath(self.opts.data_dir)
+ self.opts.CONFIG_DATA_PATH = self.opts.data_dir + "/config.data"
+ self.opts.SERVER_PEM_PATH = self.opts.data_dir + "/server.pem"
+ self.opts.CLIENT_PEM_PATH = self.opts.data_dir + "/client.pem"
+ self.opts.KNOWN_HOSTS_PATH = self.opts.data_dir + "/known_hosts"
+ self.opts.RAPI_CERT_FILE = self.opts.data_dir + "/rapi.pem"
+ self.opts.SPICE_CERT_FILE = self.opts.data_dir + "/spice.pem"
+ self.opts.SPICE_CACERT_FILE = self.opts.data_dir + "/spice-ca.pem"
+ self.opts.RAPI_USERS_FILE = self.opts.data_dir + "/rapi/users"
+ self.opts.RAPI_USERS_FILE_PRE24 = self.opts.data_dir + "/rapi_users"
+ self.opts.CONFD_HMAC_KEY = self.opts.data_dir + "/hmac.key"
+ self.opts.CDS_FILE = self.opts.data_dir + "/cluster-domain-secret"
+ self.opts.SSCONF_MASTER_NODE = self.opts.data_dir + "/ssconf_master_node"
+ self.opts.WATCHER_STATEFILE = self.opts.data_dir + "/watcher.data"
+ self.opts.FILE_STORAGE_PATHS_FILE = (self.opts.conf_dir +
+ "/file-storage-paths")
+
+ def _AskUser(self):
+ if not self.opts.force:
+ if self.opts.downgrade:
+ usertext = ("The configuration is going to be DOWNGRADED "
+ "to version %s.%s. Some configuration data might be "
+ " removed if they don't fit"
+ " in the old format. Please make sure you have read the"
+ " upgrade notes (available in the UPGRADE file and
included"
+ " in other documentation formats) to understand what they"
+ " are. Continue with *DOWNGRADING* the configuration?" %
+ (DOWNGRADE_MAJOR, DOWNGRADE_MINOR))
+ else:
+ usertext = ("Please make sure you have read the upgrade notes for"
+ " Ganeti %s (available in the UPGRADE file and included"
+ " in other documentation formats). Continue with
upgrading"
+ " configuration?" % constants.RELEASE_VERSION)
+ if not cli.AskUser(usertext):
+ sys.exit(constants.EXIT_FAILURE)
+
+ def _Downgrade(self, config_major, config_minor, config_version,
+ config_revision):
+ if not ((config_major == TARGET_MAJOR and config_minor == TARGET_MINOR) or
+ (config_major == DOWNGRADE_MAJOR and
+ config_minor == DOWNGRADE_MINOR)):
+ raise Error("Downgrade supported only from the latest version (%s.%s),"
+ " found %s (%s.%s.%s) instead" %
+ (TARGET_MAJOR, TARGET_MINOR, config_version, config_major,
+ config_minor, config_revision))
+ if not self.DowngradeAll():
+ raise Error("Downgrade failed:\n%s" % "\n".join(self.errors))
+
+ def _TestLoadingConfigFile(self):
+ # test loading the config file
+ all_ok = True
+ if not (self.opts.dry_run or self.opts.no_verify):
+ logging.info("Testing the new config file...")
+ cfg = config.ConfigWriter(cfg_file=self.opts.CONFIG_DATA_PATH,
+ accept_foreign=self.opts.ignore_hostname,
+ offline=True)
+ # if we reached this, it's all fine
+ vrfy = cfg.VerifyConfig()
+ if vrfy:
+ logging.error("Errors after conversion:")
+ for item in vrfy:
+ logging.error(" - %s", item)
+ all_ok = False
+ else:
+ logging.info("File loaded successfully after upgrading")
+ del cfg
+
+ if self.opts.downgrade:
+ action = "downgraded"
+ out_ver = "%s.%s" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR)
+ else:
+ action = "upgraded"
+ out_ver = constants.RELEASE_VERSION
+ if all_ok:
+ cli.ToStderr("Configuration successfully %s to version %s.",
+ action, out_ver)
+ else:
+ cli.ToStderr("Configuration %s to version %s, but there are errors."
+ "\nPlease review the file.", action, out_ver)
diff --cc src/Ganeti/Config.hs
index 5be157b,1076881..ddbb7b1
--- a/src/Ganeti/Config.hs
+++ b/src/Ganeti/Config.hs
@@@ -331,19 -318,14 +331,19 @@@ getGroupInstances cfg gname
getFilledInstHvParams :: [String] -> ConfigData -> Instance -> HvParams
getFilledInstHvParams globals cfg inst =
-- First get the defaults of the parent
- let maybeHvName = liftM hypervisorToRaw . instHypervisor $ inst
- let hvName = instHypervisor inst
++ let maybeHvName = instHypervisor inst
hvParamMap = fromContainer . clusterHvparams $ configCluster cfg
- parentHvParams = maybe M.empty fromContainer $ M.lookup hvName
hvParamMap
+ parentHvParams =
+ maybe M.empty fromContainer (maybeHvName >>= flip M.lookup hvParamMap)
-- Then the os defaults for the given hypervisor
- osName = instOs inst
+ maybeOsName = instOs inst
osParamMap = fromContainer . clusterOsHvp $ configCluster cfg
- osHvParamMap = maybe M.empty fromContainer $ M.lookup osName osParamMap
- osHvParams = maybe M.empty fromContainer $ M.lookup hvName osHvParamMap
+ osHvParamMap =
+ maybe M.empty (maybe M.empty fromContainer . flip M.lookup osParamMap)
+ maybeOsName
+ osHvParams =
+ maybe M.empty (maybe M.empty fromContainer . flip M.lookup
osHvParamMap)
+ maybeHvName
-- Then the child
childHvParams = fromContainer . instHvparams $ inst
-- Helper function
diff --cc src/Ganeti/Query/Instance.hs
index efd2d7e,d4feb03..e3e55af
--- a/src/Ganeti/Query/Instance.hs
+++ b/src/Ganeti/Query/Instance.hs
@@@ -894,9 -873,9 +894,9 @@@ consoleParamsToCalls params
-- instances.
getHypervisorSpecs :: ConfigData -> [Instance] -> [(Hypervisor, HvParams)]
getHypervisorSpecs cfg instances =
- let hvs = nub . map instHypervisor $ instances
+ let hvs = nub . mapMaybe instHypervisor $ instances
hvParamMap = (fromContainer . clusterHvparams . configCluster $ cfg)
- in zip hvs . map ((Map.!) hvParamMap . hypervisorToRaw) $ hvs
+ in zip hvs . map ((Map.!) hvParamMap) $ hvs
-- | Collect live data from RPC query if enabled.
collectLiveData :: Bool -- ^ Live queries allowed
diff --cc src/Ganeti/WConfd/Ssconf.hs
index 7d77079,6aa7765..8e4138f
--- a/src/Ganeti/WConfd/Ssconf.hs
+++ b/src/Ganeti/WConfd/Ssconf.hs
@@@ -42,11 -42,11 +42,12 @@@ module Ganeti.WConfd.Sscon
, mkSSConf
) where
- import Control.Arrow ((&&&))
+ import Control.Arrow ((&&&), first, second)
import Data.Foldable (Foldable(..), toList)
import Data.List (partition)
+import Data.Maybe (mapMaybe)
import qualified Data.Map as M
+ import qualified Text.JSON as J
import Ganeti.BasicTypes
import Ganeti.Config
--
Klaus Aehlig
Google Germany GmbH, Dienerstr. 12, 80331 Muenchen
Registergericht und -nummer: Hamburg, HRB 86891
Sitz der Gesellschaft: Hamburg
Geschaeftsfuehrer: Graham Law, Christine Elizabeth Flores
