Giuseppe Lavagetto has submitted this change and it was merged. Change subject: conftool: adding the cli-tool and integration tests ......................................................................
conftool: adding the cli-tool and integration tests Change-Id: I87147db34746d9f9b02cc93d97fecaec69aa6912 --- M conftool/__init__.py A conftool/action.py M conftool/cli/syncer.py A conftool/cli/tool.py M conftool/configuration.py M conftool/drivers/__init__.py M conftool/drivers/etcd.py M conftool/node.py A conftool/tests/__init__.py R conftool/tests/fixtures/nodes/eqiad.yaml R conftool/tests/fixtures/services/data.yaml A conftool/tests/integration/__init__.py A conftool/tests/integration/test_syncer.py A conftool/tests/integration/test_tool.py A conftool/tests/unit/__init__.py A conftool/tests/unit/test_node.py A setup.py 17 files changed, 534 insertions(+), 23 deletions(-) Approvals: Giuseppe Lavagetto: Verified; Looks good to me, approved diff --git a/conftool/__init__.py b/conftool/__init__.py index 1ba4c8a..36aea6d 100644 --- a/conftool/__init__.py +++ b/conftool/__init__.py @@ -1,5 +1,6 @@ import os import logging +import json from conftool import backend from conftool import drivers @@ -22,6 +23,10 @@ def key(self): raise NotImplementedError("All kvstore objects should implement this") + @property + def name(self): + return os.path.basename(self.key) + def get_default(self, what): raise NotImplementedError("All kvstore objects should implement this.") @@ -43,6 +48,23 @@ def delete(self): self.backend.driver.delete(self.key) + @classmethod + def get_tags(cls, taglist): + tuplestrip = lambda tup: tuple(map(lambda x: x.strip(), tup)) + tagdict = dict([tuplestrip(el.split('=')) for el in taglist]) + # will raise a KeyError if not all tags are matched + return [tagdict[t] for t in cls._tags] + + def update(self, values): + """ + Update values of properties in the schema + """ + for k, v in values.items(): + if k not in self._schema: + continue + self._set_value(k, self._schema[k], {k: v}, set_defaults=False) + self.write() + def _from_net(self, values): """ Fetch the values from the kvstore into the object @@ -59,9 +81,14 @@ values[key] = self.get_default(key) return values - def _set_value(self, key, validator, values): + def _set_value(self, key, validator, values, set_defaults=True): try: setattr(self, key, validator(values[key])) except Exception as e: # TODO: log validation error - setattr(self, key, self.get_default(key)) + if set_defaults: + setattr(self, key, self.get_default(key)) + + def __str__(self): + d = {self.name: self._to_net()} + return json.dumps(d) diff --git a/conftool/action.py b/conftool/action.py new file mode 100644 index 0000000..2f73aa8 --- /dev/null +++ b/conftool/action.py @@ -0,0 +1,50 @@ +config = {} +backend = None + + +class ActionError(Exception): + pass + + +class Action(object): + + def __init__(self, obj, act): + self.action, self.args = self._parse_action(act) + self.entity = obj + self.description = "" + + def _parse_action(self, act): + if act.startswith('get'): + return ('get', None) + elif act.startswith('delete'): + return ('delete', None) + elif not act.startswith('set/'): + raise ActionError("Cannot parse action %s" % act) + set_arg = act.strip('set/') + try: + values = dict((el.strip().split('=')) for el in set_arg.split(':')) + except Exception as e: + raise ActionError("Could not parse set instructions: %s" % set_arg) + return ('set', values) + + def run(self): + if self.action == 'get': + self.entity.fetch() + if self.entity.exists: + return str(self.entity) + else: + return "%s not found" % self.entity.name + elif self.action == 'delete': + self.entity.delete() + entity_type = self.entity.__class__.__name__, + return "Deleted %s %s." % (entity_type, + self.entity.name) + else: + desc = [] + for (k, v) in self.args.items(): + msg = "%s: %s changed %s => %s" % ( + self.entity.name, k, + getattr(self.entity, k), v) + desc.append(msg) + self.entity.update(self.args) + return "\n".join(desc) diff --git a/conftool/cli/syncer.py b/conftool/cli/syncer.py index 73c797d..486cc4c 100644 --- a/conftool/cli/syncer.py +++ b/conftool/cli/syncer.py @@ -10,6 +10,7 @@ import functools import logging + # Generic exception handling decorator def catch_and_log(log_msg): def actual_wrapper(fn): @@ -45,7 +46,7 @@ _log.debug("New services in cluster %s: %s", cluster, " ".join(new_services)) _log.debug("Services to remove in cluster %s: %s", cluster, - " ".join(new_services)) + " ".join(del_services)) return (new_services | changed_services, del_services) @@ -127,25 +128,32 @@ delete_node(dc, cluster, servname, el) -def tag_files(l): - servicefile = None - poolsfiles = [] - for filename in l: - if filename.endswith('services.yaml'): - servicefile = filename - else: - poolsfiles.append(filename) - return servicefile, poolsfiles +def tag_files(directory): + def tag(d, path, files): + tag = path.replace(directory, '').lstrip('/') + if not tag: + return + real_files = [os.path.realpath(os.path.join(path, f)) for f in files] + d[tag].extend(real_files) + tagged = defaultdict(list) + os.path.walk(directory, tag, tagged) + return tagged - -def main(): +def get_args(args): parser = argparse.ArgumentParser(description="Tool to sync the declared " "configuration on-disk with the kvstore " "data") - parser.add_argument('--files', nargs='+', help="Location (on disk) of the" - " file(s) that have changed") + parser.add_argument('--directory', help="Directory containing the files to sync") parser.add_argument('--config', help="Configuration file") - args = parser.parse_args() + return parser.parse_args(args) + + +def main(arguments=None): + if arguments is None: + arguments = list(sys.argv) + arguments.pop(0) + + args = get_args(arguments) logging.basicConfig(level=logging.DEBUG) try: c = configuration.get(args.config) @@ -154,9 +162,11 @@ raise _log.critical("Invalid configuration: %s", e) sys.exit(1) - (service_file, pools) = tag_files(args.files) - # Load services data - if service_file is not None: + files = tag_files(args.directory) + # Load services data. + # TODO: This must be fixed to be less specific + if files['services']: + service_file = files['services'][0] with open(service_file, 'rb') as fh: servdata = yaml.load(fh) else: @@ -167,7 +177,7 @@ load, rem[cluster] = get_service_actions(cluster, data) load_services(cluster, load, data) # sync nodes - for filename in pools: + for filename in files['nodes']: dc = os.path.basename(filename).rstrip('.yaml') with open(filename, 'rb') as fh: dc_data = yaml.load(fh) diff --git a/conftool/cli/tool.py b/conftool/cli/tool.py new file mode 100644 index 0000000..a18127d --- /dev/null +++ b/conftool/cli/tool.py @@ -0,0 +1,76 @@ +# Conftool cli module +# +import argparse +import sys +import logging +import json +from conftool import configuration, action, _log, KVObject +from conftool.drivers import BackendError +# TODO: auto import these somehow +from conftool import service, node + +object_types = {"node": node.Node, "service": service.Service} + +def main(cmdline=None): + if cmdline is None: + cmdline = list(sys.argv) + cmdline.pop(0) + + parser = argparse.ArgumentParser( + description="Tool to interact with the WMF config store", + epilog="More details at" + " <https://wikitech.wikimedia.org/wiki/conftool>.", + fromfile_prefix_chars='@') + parser.add_argument('--config', nargs=1, help="Optional config file", + default="/etc/conftool/config") + parser.add_argument('--tags', + help="List of comma-separated tags; they need to " + "match the base tags of the object type you chose.", + required=True) + parser.add_argument('--object-type', dest="object_type", + choices=object_types.keys(), default='node') + parser.add_argument('--action', action="append", metavar="ACTIONS", + help="the action to take: " + " [set/k1=v1:k2=v2...|get|delete] node", nargs=2, + required=True) + args = parser.parse_args(cmdline) + logging.basicConfig(level=logging.WARN) + try: + c = configuration.get(args.config) + KVObject.setup(c) + except Exception as e: + _log.critical("Invalid configuration: %s", e) + sys.exit(1) + + cls = object_types[args.object_type] + try: + tags = cls.get_tags(args.tags.split(',')) + except KeyError as e: + _log.critical("Invalid tag list %s - reason: %s", args.tags, e) + sys.exit(1) + + for unit in args.action: + try: + act, name = unit + if act == 'get' and name == "all": + cur_dir = cls.dir(*tags) + print json.dumps(dict(KVObject.backend.driver.ls(cur_dir))) + return + # Oh python I <3 you... + arguments = list(tags) + arguments.append(name) + obj = cls(*arguments) + a = action.Action(obj, act) + msg = a.run() + except action.ActionError as e: + _log.error("Invalid action, reason: %s", str(e)) + except BackendError as e: + _log.error("Failure writing to the kvstore: %s", str(e)) + except Exception as e: + _log.error("Generic action failure: %s", str(e)) + else: + print(msg) + + +if __name__ == '__main__': + main() diff --git a/conftool/configuration.py b/conftool/configuration.py index 216452b..c3fede6 100644 --- a/conftool/configuration.py +++ b/conftool/configuration.py @@ -17,6 +17,7 @@ ConfigBase = collections.namedtuple('Config', ['driver', 'hosts', 'namespace', + 'api_version', 'pools_path', 'services_path', ]) @@ -27,6 +28,7 @@ driver='etcd', hosts=['http://localhost:2379'], namespace='/conftool', + api_version='v1', pools_path='pools', services_path='services' ): @@ -38,5 +40,6 @@ return super(Config, cls).__new__(cls, driver=driver, hosts=hosts, namespace=namespace, + api_version=api_version, pools_path=pools_path, services_path=services_path) diff --git a/conftool/drivers/__init__.py b/conftool/drivers/__init__.py index 40a2ba9..35e7b48 100644 --- a/conftool/drivers/__init__.py +++ b/conftool/drivers/__init__.py @@ -9,7 +9,7 @@ class BaseDriver(object): def __init__(self, config): - self.base_path = config.namespace + self.base_path = os.path.join(config.namespace, config.api_version) def abspath(self, path): if path.startswith('/'): diff --git a/conftool/drivers/etcd.py b/conftool/drivers/etcd.py index 11e812d..b686c3a 100644 --- a/conftool/drivers/etcd.py +++ b/conftool/drivers/etcd.py @@ -67,7 +67,6 @@ except etcd.EtcdKeyNotFound: return None - def _data(self, etcdresult): if etcdresult is None or etcdresult.dir: return None diff --git a/conftool/node.py b/conftool/node.py index 9262a84..4dfedd9 100644 --- a/conftool/node.py +++ b/conftool/node.py @@ -13,6 +13,7 @@ class Node(KVObject): _schema = {'weight': int, 'pooled': choice("yes", "no", "inactive")} + _tags = ['dc', 'cluster', 'service'] def __init__(self, datacenter, cluster, servname, host): self.base_path = self.config.pools_path diff --git a/conftool/tests/__init__.py b/conftool/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/conftool/tests/__init__.py diff --git a/conftool/tests/fixtures/pools/eqiad.yaml b/conftool/tests/fixtures/nodes/eqiad.yaml similarity index 100% rename from conftool/tests/fixtures/pools/eqiad.yaml rename to conftool/tests/fixtures/nodes/eqiad.yaml diff --git a/conftool/tests/fixtures/services.yaml b/conftool/tests/fixtures/services/data.yaml similarity index 100% rename from conftool/tests/fixtures/services.yaml rename to conftool/tests/fixtures/services/data.yaml diff --git a/conftool/tests/integration/__init__.py b/conftool/tests/integration/__init__.py new file mode 100644 index 0000000..22bbf9e --- /dev/null +++ b/conftool/tests/integration/__init__.py @@ -0,0 +1,103 @@ +import os +import time +import logging +import subprocess +import unittest +import shutil +import tempfile +from conftool import configuration, KVObject + +test_base = os.path.realpath(os.path.join( + os.path.dirname(__file__), '..')) + + +class EtcdProcessHelper(object): + + def __init__( + self, + base_directory, + proc_name='etcd', + port=2379, + internal_port=2380, + cluster=False, + tls=False + ): + + self.base_directory = base_directory + self.proc_name = proc_name + self.port = port + self.internal_port = internal_port + self.proc = None + self.cluster = cluster + self.schema = 'http://' + if tls: + self.schema = 'https://' + + def run(self, proc_args=None): + log = logging.getLogger() + if self.proc is not None: + raise Exception("etcd already running with pid %d", self.proc.pid) + client = '%s127.0.0.1:%d' % (self.schema, self.port) + daemon_args = [ + self.proc_name, + '-data-dir', self.base_directory, + '-name', 'test-node', + '-advertise-client-urls', client, + '-listen-client-urls', client + ] + if proc_args: + daemon_args.extend(proc_args) + + daemon = subprocess.Popen(daemon_args) + log.debug('Started %d' % daemon.pid) + log.debug('Params: %s' % daemon_args) + time.sleep(2) + self.proc = daemon + + def stop(self): + self.proc.kill() + self.proc = None + + +class IntegrationTestBase(unittest.TestCase): + @classmethod + def setUpClass(cls): + program = cls._get_exe() + cls.directory = tempfile.mkdtemp(prefix='conftool') + cls.processHelper = EtcdProcessHelper( + cls.directory, + proc_name=program, port=2379) + cls.processHelper.run() + cls.fixture_dir = os.path.join(test_base, 'fixtures') + conf = configuration.get(None) + KVObject.setup(conf) + + @classmethod + def tearDownClass(cls): + cls.processHelper.stop() + shutil.rmtree(cls.directory) + + @classmethod + def _is_exe(cls, fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + @classmethod + def _get_exe(cls): + PROGRAM = 'etcd' + program_path = None + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, PROGRAM) + if cls._is_exe(exe_file): + program_path = exe_file + break + if not program_path: + raise Exception('etcd not in path!!') + return program_path + + def tearDown(self): + path = KVObject.backend.driver.base_path + try: + KVObject.backend.driver.client.delete(path, recursive=True, dir=True) + except: + pass diff --git a/conftool/tests/integration/test_syncer.py b/conftool/tests/integration/test_syncer.py new file mode 100644 index 0000000..ca1a220 --- /dev/null +++ b/conftool/tests/integration/test_syncer.py @@ -0,0 +1,117 @@ +import os +from conftool.cli import syncer +from conftool import service, node +from conftool.tests.integration import IntegrationTestBase, test_base + + +class SyncerIntegration(IntegrationTestBase): + + @staticmethod + def service_generator(base_servname, number, initial=0): + data = {} + for i in xrange(initial, number): + servname = base_servname + str(i) + data[servname] = {'default_values': {"pooled": "yes", "weight": i}, + 'datacenters': ['kitchen_sink', 'sofa']} + return data + + @staticmethod + def nodelist_generator(servnames, number, initial=0): + return {nodename: servnames for nodename + in ["node-%d" % i for i in xrange(initial, number)]} + + def node_generator(self, cluster, servnames, number, initial=0): + return {cluster: self.nodelist_generator(servnames, number, initial)} + + def test_tag_files(self): + d = os.path.join(test_base,'fixtures') + res = syncer.tag_files(d) + self.assertEquals( + res['services'], [os.path.join(d,'services/data.yaml')]) + + def test_load_service(self): + cluster = 'test' + servname = 'espresso-machine' + data = {'default_values': {"pooled": "yes", "weight": 0}, + 'datacenters': ['kitchen_sink', 'sofa']} + syncer.load_service(cluster, servname, data) + s = service.Service(cluster, servname) + self.assertEquals(s.default_values["pooled"], "yes") + self.assertEquals(s.datacenters[0], 'kitchen_sink') + + def test_load_services(self): + cluster = 'test' + data = self.service_generator('espresso-machine', 10) + syncer.load_services(cluster, data.keys(), data) + for i in xrange(10): + servname = 'espresso-machine' + str(i) + s = service.Service(cluster, servname) + self.assertEquals(s.default_values, data[servname]['default_values']) + + + def test_remove_services(self): + cluster = 'test' + data = self.service_generator('espresso-machine', 10) + syncer.load_services(cluster, data.keys(), data) + del data['espresso-machine0'] + syncer.remove_services(cluster, data.keys()) + s = service.Service(cluster, 'espresso-machine0') + self.assertTrue(s.exists) + for i in xrange(1,10): + servname = 'espresso-machine' + str(i) + s = service.Service(cluster, servname) + self.assertFalse(s.exists) + + def test_get_service_actions(self): + cluster = 'test' + data = self.service_generator('espresso-machine', 10) + syncer.load_services(cluster, data.keys(), data) + new_data = self.service_generator('espresso-machine', 15, initial=5) + new_data['espresso-machine6']['datacenters'] = ['sofa'] + (new, delete) = syncer.get_service_actions(cluster, new_data) + # Pick one machine that is new + self.assertIn('espresso-machine12', new) + # one removed + self.assertIn('espresso-machine4', delete) + # one modified + self.assertIn('espresso-machine6', new) + # one not modified at all + self.assertNotIn('espresso-machine7', new) + self.assertNotIn('espresso-machine7', delete) + + def test_load_node(self): + cluster = 'test' + sdata = self.service_generator('espresso-machine', 2, initial=1) + syncer.load_services(cluster, sdata.keys(), sdata) + serv = sdata.keys().pop() + syncer.load_node('sofa', cluster, serv, 'one-off') + n = node.Node('sofa', cluster, serv, 'one-off') + self.assertTrue(n.exists) + self.assertEquals(n.weight, sdata[serv]['default_values']['weight']) + + def test_get_changed_nodes(self): + dc = 'sofa' + cluster = 'test' + sdata = self.service_generator('espresso-machine', 2, 1) + syncer.load_services(cluster, sdata.keys(), sdata) + for i in xrange(10): + syncer.load_node(dc,cluster, 'espresso-machine1', 'node-%d' % i) + expected_hosts = ["node-%d" % i for i in xrange(5,15)] + n, d = syncer.get_changed_nodes(dc, cluster, + 'espresso-machine1', expected_hosts) + self.assertIn('node-13', n) + self.assertIn('node-4', d) + + def test_load_nodes(self): + dc = 'sofa' + cluster = 'test' + sdata = self.service_generator('espresso-machine', 2) + syncer.load_services(cluster, sdata.keys(), sdata) + data = self.node_generator(cluster, sdata.keys(), 20) + syncer.load_nodes(dc, data) + for servname in sdata.keys(): + for i in xrange(20): + nodename = "node-%d" % i + n = node.Node(dc, cluster, servname, nodename) + self.assertTrue(n.exists) + self.assertEquals(n.pooled, 'yes') diff --git a/conftool/tests/integration/test_tool.py b/conftool/tests/integration/test_tool.py new file mode 100644 index 0000000..2effe51 --- /dev/null +++ b/conftool/tests/integration/test_tool.py @@ -0,0 +1,42 @@ +import os +import sys +from conftool.cli import syncer, tool +from conftool import service, node +from conftool.tests.integration import IntegrationTestBase, test_base +from contextlib import contextmanager +from StringIO import StringIO +import json + +@contextmanager +def captured_output(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + +class SyncerIntegration(IntegrationTestBase): + + def setUp(self): + args = ['--directory', os.path.join(test_base, 'fixtures')] + syncer.main(arguments=args) + + def generate_args(self, *actions): + args = ['--tags', "dc=eqiad,cluster=cache_text,service=https"] + for action in actions: + args.append('--action') + args.extend(action.split()) + return args + + def test_get_node(self): + args = self.generate_args('get cp1008') + print args + with captured_output() as (out, err): + tool.main(cmdline=args) + res = out.getvalue().strip() + output = json.loads(res) + self.assertEquals(output.keys(), ['cp1008']) + self.assertEquals(output['cp1008']['pooled'], 'no') diff --git a/conftool/tests/unit/__init__.py b/conftool/tests/unit/__init__.py new file mode 100644 index 0000000..4b757a9 --- /dev/null +++ b/conftool/tests/unit/__init__.py @@ -0,0 +1 @@ +from . import test_node diff --git a/conftool/tests/unit/test_node.py b/conftool/tests/unit/test_node.py new file mode 100644 index 0000000..0dd285f --- /dev/null +++ b/conftool/tests/unit/test_node.py @@ -0,0 +1,60 @@ +import unittest +import mock +import conftool +from conftool import KVObject, node, service +from conftool import configuration, drivers, backend + +class MockDriver(drivers.BaseDriver): + def __init__(self, config): + self.base_path = '/base_path/v2' + +class MockBackend(object): + def __init__(self, config): + self.config = config + self.driver = MockDriver(config) + +class TestNode(unittest.TestCase): + + def _mock_read(self, values): + KVObject.backend.driver.read = mock.MagicMock(return_value=values) + + def _mock_defaults(self, values): + def _side_effect(what): + return values[what] + service.Service.get_defaults = mock.MagicMock(side_effect=_side_effect) + + def setUp(self): + KVObject.backend = MockBackend({}) + KVObject.config = configuration.Config(driver="") + pass + + @mock.patch('conftool.node.Node.get_default') + def test_new_node(self, mocker): + """New node creation""" + mocker.return_value = 'default_value' + self._mock_read(None) + n = node.Node('dc', 'cluster', 'service', 'foo') + # Test + self.assertEquals(n.base_path, 'pools') + self.assertEquals(n.key, 'pools/dc/cluster/service/foo') + self.assertFalse(n.exists) + self.assertEquals(n.pooled, 'default_value') + self.assertEquals(n.name, 'foo') + + def test_read(self): + """Test that reading fetches correctly the values""" + self._mock_read({"pooled": "yes", "weight": 20}) + n = node.Node('dc', 'cluster', 'service', 'foo') + self.assertEquals(n.weight, 20) + self.assertEquals(n.pooled, "yes") + + def test_failed_validation(self): + """Test bad validation""" + self._mock_read({"pooled": "maybe?", "weight": 20}) + n = node.Node('dc', 'cluster', 'service', 'foo') + self.assertEquals(n.pooled, "no") + # Note: this fails at the moment + # self.assertRaises(ValueError, setattr, n, "pooled", "maybe") + + def test_dir(self): + self.assertEquals(node.Node.dir('a', 'b', 'c'), 'pools/a/b/c') diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..ec3dcce --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/python + +from setuptools import setup, find_packages + +setup( + name='conftool', + version='0.0.1', + description='Collection of tools to interoperate with distributed k/v stores', + author='Joe', + author_email='glavage...@wikimedia.org', + url='https://github.com/wikimedia/operations-software-conftool', + install_requires=['python-etcd', 'yaml'], + setup_requires=[], + zip_safe=True, + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'conftool-sync = conftool.cli.syncer:main', + 'confctl = conftool.cli.tool:main', + ], + }, +) -- To view, visit https://gerrit.wikimedia.org/r/215891 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I87147db34746d9f9b02cc93d97fecaec69aa6912 Gerrit-PatchSet: 5 Gerrit-Project: operations/software/conftool Gerrit-Branch: master Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits