commit 4b7ebadfe243ec034d4d4de2187e0d0bf068abb4
Merge: ef30bed 16e9c03
Author: Helga Velroyen <[email protected]>
Date: Wed Mar 18 17:53:25 2015 +0100
Merge branch 'stable-2.12' into stable-2.13
* stable-2.11
Renew crypto retries for non-master nodes
Retries for the master's SSL cert renewal
Unit tests for offline nodes
De-duplicate testing code regarding pathutils
Make LURenewCrypto handle unreachable nodes properly
Error handling on failed SSL cert renewal for master
Unit test for LURenewCrypto's valid case
Mock support for pathutils
Increase timeout of crypto token RPC
Conflicts:
lib/cmdlib/cluster.py
Resolution:
Apply branching in LURenewCrypto based on flags.
Signed-off by: Helga Velroyen <[email protected]>
diff --cc lib/cmdlib/cluster.py
index 58007f9,828212f..291cf47
--- a/lib/cmdlib/cluster.py
+++ b/lib/cmdlib/cluster.py
@@@ -106,37 -106,13 +106,38 @@@ def _UpdateMasterClientCert
class LUClusterRenewCrypto(NoHooksLU):
"""Renew the cluster's crypto tokens.
- Note that most of this operation is done in gnt_cluster.py, this LU only
- takes care of the renewal of the client SSL certificates.
-
"""
+
+ _MAX_NUM_RETRIES = 3
+ REQ_BGL = False
- def Exec(self, feedback_fn):
+ def ExpandNames(self):
+ self.needed_locks = {
+ locking.LEVEL_NODE: locking.ALL_SET,
+ locking.LEVEL_NODE_ALLOC: locking.ALL_SET,
+ }
+ self.share_locks = ShareAll()
+ self.share_locks[locking.LEVEL_NODE] = 0
+ self.share_locks[locking.LEVEL_NODE_ALLOC] = 0
+
+ def CheckPrereq(self):
+ """Check prerequisites.
+
+ This checks whether the cluster is empty.
+
+ Any errors are signaled by raising errors.OpPrereqError.
+
+ """
+ self._ssh_renewal_suppressed = \
+ not self.cfg.GetClusterInfo().modify_ssh_setup and self.op.ssh_keys
+
- def _RenewNodeSslCertificates(self):
++ def _RenewNodeSslCertificates(self, feedback_fn):
+ """Renews the nodes' SSL certificates.
+
+ Note that most of this operation is done in gnt_cluster.py, this LU
only
+ takes care of the renewal of the client SSL certificates.
+
+ """
master_uuid = self.cfg.GetMasterNode()
server_digest = utils.GetCertificateDigest(
@@@ -157,47 -155,33 +180,65 @@@
nodes = self.cfg.GetAllNodesInfo()
for (node_uuid, node_info) in nodes.items():
if node_info.offline:
- feedback_fn("* Skipping offline node %s" % node_info.name)
+ logging.info("* Skipping offline node %s", node_info.name)
continue
if node_uuid != master_uuid:
- new_digest = CreateNewClientCert(self, node_uuid)
- if node_info.master_candidate:
- self.cfg.AddNodeToCandidateCerts(node_uuid, new_digest)
+ for _ in range(self._MAX_NUM_RETRIES):
+ try:
+ new_digest = CreateNewClientCert(self, node_uuid)
+ if node_info.master_candidate:
+ self.cfg.AddNodeToCandidateCerts(node_uuid,
+ new_digest)
+ break
+ except errors.OpExecError as last_exception:
+ pass
+ else:
+ if last_exception:
+ node_errors[node_uuid] = last_exception
+
+ if node_errors:
+ msg = ("Some nodes' SSL client certificates could not be renewed."
+ " Please make sure those nodes are reachable and rerun"
+ " the operation. The affected nodes and their errors are:\n")
+ for uuid, e in node_errors.items():
+ msg += "Node %s: %s\n" % (uuid, e)
+ feedback_fn(msg)
+
self.cfg.RemoveNodeFromCandidateCerts("%s-SERVER" % master_uuid)
self.cfg.RemoveNodeFromCandidateCerts("%s-OLDMASTER" % master_uuid)
+ def _RenewSshKeys(self):
+ """Renew all nodes' SSH keys.
+
+ """
+ master_uuid = self.cfg.GetMasterNode()
+
+ nodes = self.cfg.GetAllNodesInfo()
+ nodes_uuid_names = [(node_uuid, node_info.name) for (node_uuid,
node_info)
+ in nodes.items() if not node_info.offline]
+ node_names = [name for (_, name) in nodes_uuid_names]
+ node_uuids = [uuid for (uuid, _) in nodes_uuid_names]
+ port_map = ssh.GetSshPortMap(node_names, self.cfg)
+ potential_master_candidates = self.cfg.GetPotentialMasterCandidates()
+ master_candidate_uuids = self.cfg.GetMasterCandidateUuids()
+ result = self.rpc.call_node_ssh_keys_renew(
+ [master_uuid],
+ node_uuids, node_names, port_map,
+ master_candidate_uuids,
+ potential_master_candidates)
+ result[master_uuid].Raise("Could not renew the SSH keys of all nodes")
+
+ def Exec(self, feedback_fn):
+ if self.op.node_certificates:
+ feedback_fn("Renewing Node SSL certificates")
- self._RenewNodeSslCertificates()
++ self._RenewNodeSslCertificates(feedback_fn)
+ if self.op.ssh_keys and not self._ssh_renewal_suppressed:
+ feedback_fn("Renewing SSH keys")
+ self._RenewSshKeys()
+ elif self._ssh_renewal_suppressed:
+ feedback_fn("Cannot renew SSH keys if the cluster is configured to
not"
+ " modify the SSH setup.")
+
class LUClusterActivateMasterIp(NoHooksLU):
"""Activate the master IP on the master node.
diff --cc test/py/cmdlib/cluster_unittest.py
index 1e182cd,eaed548..ee4e759
--- a/test/py/cmdlib/cluster_unittest.py
+++ b/test/py/cmdlib/cluster_unittest.py
@@@ -2273,5 -2253,252 +2275,252 @@@ class TestLUClusterVerifyDisks(CmdlibTe
self.assertEqual(1, len(result["jobs"]))
+ class TestLUClusterRenewCrypto(CmdlibTestCase):
+
+ def setUp(self):
+ super(TestLUClusterRenewCrypto, self).setUp()
+ self._node_cert = self._CreateTempFile()
+ shutil.copy(testutils.TestDataFilename("cert1.pem"), self._node_cert)
+ self._client_node_cert = self._CreateTempFile()
+ shutil.copy(testutils.TestDataFilename("cert2.pem"),
self._client_node_cert)
+ self._client_node_cert_tmp = self._CreateTempFile()
+
+ def tearDown(self):
+ super(TestLUClusterRenewCrypto, self).tearDown()
+
+ def _GetFakeDigest(self, uuid):
+ """Creates a fake SSL digest depending on the UUID of a node.
+
+ @type uuid: string
+ @param uuid: node UUID
+ @returns: a string impersonating a SSL digest
+
+ """
+ return "FA:KE:%s:%s:%s:%s" % (uuid[0:2], uuid[2:4], uuid[4:6],
uuid[6:8])
+
+ def _InitPathutils(self, pathutils):
+ """Patch pathutils to point to temporary files.
+
+ """
+ pathutils.NODED_CERT_FILE = self._node_cert
+ pathutils.NODED_CLIENT_CERT_FILE = self._client_node_cert
+ pathutils.NODED_CLIENT_CERT_FILE_TMP = \
+ self._client_node_cert_tmp
+
+ def _AssertCertFiles(self, pathutils):
+ """Check if the correct certificates exist and don't exist on the
master.
+
+ """
+ self.assertTrue(os.path.exists(pathutils.NODED_CERT_FILE))
+ self.assertTrue(os.path.exists(pathutils.NODED_CLIENT_CERT_FILE))
+ self.assertFalse(os.path.exists(pathutils.NODED_CLIENT_CERT_FILE_TMP))
+
+ def _CompletelySuccessfulRpc(self, node_uuid, _):
+ """Fake RPC call which always returns successfully.
+
+ """
+ return self.RpcResultsBuilder() \
+ .CreateSuccessfulNodeResult(node_uuid,
+ [(constants.CRYPTO_TYPE_SSL_DIGEST,
+ self._GetFakeDigest(node_uuid))])
+
+ @patchPathutils("cluster")
+ def testSuccessfulCase(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ # create a few non-master, online nodes
+ num_nodes = 3
+ for _ in range(num_nodes):
+ self.cfg.AddNewNode()
+ self.rpc.call_node_crypto_tokens = self._CompletelySuccessfulRpc
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ # Check if we have the correct digests in the configuration
+ cluster = self.cfg.GetClusterInfo()
+ self.assertEqual(num_nodes + 1, len(cluster.candidate_certs))
+ nodes = self.cfg.GetAllNodesInfo()
+ for (node_uuid, _) in nodes.items():
+ expected_digest = self._GetFakeDigest(node_uuid)
+ self.assertEqual(expected_digest,
cluster.candidate_certs[node_uuid])
+
+ @patchPathutils("cluster")
+ def testMasterFails(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ # make sure the RPC calls are failing for all nodes
+ master_uuid = self.cfg.GetMasterNode()
+ self.rpc.call_node_crypto_tokens.return_value =
self.RpcResultsBuilder() \
+ .CreateFailedNodeResult(master_uuid)
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ # Check if we correctly have no candidate certificates
+ cluster = self.cfg.GetClusterInfo()
+ self.assertFalse(cluster.candidate_certs)
+
+ def _partiallyFailingRpc(self, node_uuid, _):
+ if node_uuid == self._failed_node:
+ return self.RpcResultsBuilder() \
+ .CreateFailedNodeResult(node_uuid)
+ else:
+ return self.RpcResultsBuilder() \
+ .CreateSuccessfulNodeResult(node_uuid,
+ [(constants.CRYPTO_TYPE_SSL_DIGEST,
self._GetFakeDigest(node_uuid))])
+
+ @patchPathutils("cluster")
+ def testNonMasterFails(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ # create a few non-master, online nodes
+ num_nodes = 3
+ for _ in range(num_nodes):
+ self.cfg.AddNewNode()
+ nodes = self.cfg.GetAllNodesInfo()
+
+ # pick one node as the failing one
+ master_uuid = self.cfg.GetMasterNode()
+ self._failed_node = [node_uuid for node_uuid in nodes
+ if node_uuid != master_uuid][1]
+ self.rpc.call_node_crypto_tokens = self._partiallyFailingRpc
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ # Check if we have the correct digests in the configuration
+ cluster = self.cfg.GetClusterInfo()
+ # There should be one digest missing.
+ self.assertEqual(num_nodes, len(cluster.candidate_certs))
+ nodes = self.cfg.GetAllNodesInfo()
+ for (node_uuid, _) in nodes.items():
+ if node_uuid == self._failed_node:
+ self.assertTrue(node_uuid not in cluster.candidate_certs)
+ else:
+ expected_digest = self._GetFakeDigest(node_uuid)
+ self.assertEqual(expected_digest,
cluster.candidate_certs[node_uuid])
+
+ @patchPathutils("cluster")
+ def testOfflineNodes(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ # create a few non-master, online nodes
+ num_nodes = 3
+ offline_index = 1
+ for i in range(num_nodes):
+ # Pick one node to be offline.
+ self.cfg.AddNewNode(offline=(i==offline_index))
+ self.rpc.call_node_crypto_tokens = self._CompletelySuccessfulRpc
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ # Check if we have the correct digests in the configuration
+ cluster = self.cfg.GetClusterInfo()
+ # There should be one digest missing.
+ self.assertEqual(num_nodes, len(cluster.candidate_certs))
+ nodes = self.cfg.GetAllNodesInfo()
+ for (node_uuid, node_info) in nodes.items():
+ if node_info.offline == True:
+ self.assertTrue(node_uuid not in cluster.candidate_certs)
+ else:
+ expected_digest = self._GetFakeDigest(node_uuid)
+ self.assertEqual(expected_digest,
cluster.candidate_certs[node_uuid])
+
+ def _RpcSuccessfulAfterRetries(self, node_uuid, _):
+ if self._retries < self._max_retries:
+ self._retries += 1
+ return self.RpcResultsBuilder() \
+ .CreateFailedNodeResult(node_uuid)
+ else:
+ return self.RpcResultsBuilder() \
+ .CreateSuccessfulNodeResult(node_uuid,
+ [(constants.CRYPTO_TYPE_SSL_DIGEST,
self._GetFakeDigest(node_uuid))])
+
+ @patchPathutils("cluster")
+ def testMasterRetriesSuccess(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ self._max_retries = 2
+ self._retries = 0
+ self.rpc.call_node_crypto_tokens = self._RpcSuccessfulAfterRetries
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ cluster = self.cfg.GetClusterInfo()
+ master_uuid = self.cfg.GetMasterNode()
+ self.assertTrue(self._GetFakeDigest(master_uuid)
+ in cluster.candidate_certs.values())
+
+ @patchPathutils("cluster")
+ def testMasterRetriesFail(self, pathutils):
+ self._InitPathutils(pathutils)
+
+ self._max_retries = 5
+ self._retries = 0
+ self.rpc.call_node_crypto_tokens = self._RpcSuccessfulAfterRetries
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ cluster = self.cfg.GetClusterInfo()
+ self.assertFalse(cluster.candidate_certs)
+
+ def _RpcSuccessfulAfterRetriesNonMaster(self, node_uuid, _):
+ if self._retries < self._max_retries and node_uuid !=
self._master_uuid:
+ self._retries += 1
+ return self.RpcResultsBuilder() \
+ .CreateFailedNodeResult(node_uuid)
+ else:
+ return self.RpcResultsBuilder() \
+ .CreateSuccessfulNodeResult(node_uuid,
+ [(constants.CRYPTO_TYPE_SSL_DIGEST,
self._GetFakeDigest(node_uuid))])
+
+ def _NonMasterRetries(self, pathutils, max_retries):
+ self._InitPathutils(pathutils)
+
+ self._master_uuid = self.cfg.GetMasterNode()
+ self._max_retries = max_retries
+ self._retries = 0
+ self.rpc.call_node_crypto_tokens =
self._RpcSuccessfulAfterRetriesNonMaster
+
+ # Add one non-master node
+ self.cfg.AddNewNode()
+
- op = opcodes.OpClusterRenewCrypto()
++ op = opcodes.OpClusterRenewCrypto(node_certificates=True)
+ self.ExecOpCode(op)
+
+ self._AssertCertFiles(pathutils)
+
+ return self.cfg.GetClusterInfo()
+
+ @patchPathutils("cluster")
+ def testNonMasterRetriesSuccess(self, pathutils):
+ cluster = self._NonMasterRetries(pathutils, 2)
+ self.assertEqual(2, len(cluster.candidate_certs.values()))
+
+ @patchPathutils("cluster")
+ def testNonMasterRetriesFail(self, pathutils):
+ cluster = self._NonMasterRetries(pathutils, 5)
+
+ # Only the master digest should be in the cert list
+ self.assertEqual(1, len(cluster.candidate_certs.values()))
+ self.assertTrue(self._master_uuid in cluster.candidate_certs)
+
+
if __name__ == "__main__":
testutils.GanetiTestProgram()
