Giuseppe Lavagetto has uploaded a new change for review.
https://gerrit.wikimedia.org/r/278552
Change subject: Add select mode, refactor conftool.cli.tool
......................................................................
Add select mode, refactor conftool.cli.tool
Bug: T128199
Change-Id: I2ee04cd3e8eedf8d3f985b85099932f1d48d175b
---
M conftool/__init__.py
M conftool/cli/tool.py
M conftool/drivers/etcd.py
M conftool/tests/integration/test_tool.py
M conftool/tests/unit/test_cli_tool.py
5 files changed, 203 insertions(+), 67 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/conftool
refs/changes/52/278552/1
diff --git a/conftool/__init__.py b/conftool/__init__.py
index 19e1b26..5da9615 100644
--- a/conftool/__init__.py
+++ b/conftool/__init__.py
@@ -51,7 +51,8 @@
def tags(self):
"""Returns a dict of the current tags"""
res = {}
- # The current key, minus the basepath, is the list of tags + the node
name
+ # The current key, minus the basepath, is the list of tags +
+ # the node name
tags = self.key.replace(self.base_path(), '').split('/')[1:-1]
for i in range(len(self._tags)):
res[self._tags[i]] = tags[i]
diff --git a/conftool/cli/tool.py b/conftool/cli/tool.py
index b802955..d784b68 100644
--- a/conftool/cli/tool.py
+++ b/conftool/cli/tool.py
@@ -1,14 +1,17 @@
# Conftool cli module
#
import argparse
-import sys
+from collections import defaultdict
import logging
import json
+import os
+import re
+import sys
+
from conftool import configuration, action, _log, KVObject
from conftool.drivers import BackendError
# TODO: auto import these somehow
from conftool import service, node
-import re
class ToolCli(object):
@@ -16,11 +19,7 @@
def __init__(self, args):
self.args = args
- if self.args.tags:
- self._tags = self.args.tags.split(',')
- elif not self.args.find:
- _log.critical("Either tags or find should be provided")
- sys.exit(1)
+ self._tags = self.args.taglist.split(',')
self.entity = self.object_types[args.object_type]
def setup(self):
@@ -29,26 +28,22 @@
@property
def tags(self):
- if self.args.find:
- return []
- else:
- try:
- return self.entity.parse_tags(self._tags)
- except KeyError as e:
- _log.critical(
- "Invalid tag list %s - we're missing tag: %s",
- self.args.tags, e)
- sys.exit(1)
+ try:
+ return self.entity.parse_tags(self._tags)
+ except KeyError as e:
+ _log.critical(
+ "Invalid tag list %s - we're missing tag: %s",
+ self.args.taglist, e)
+ sys.exit(1)
+ except ValueError:
+ _log.critical("Invalid tag list %s", self.args.taglist)
+ sys.exit(1)
def host_list(self):
- if self.args.find:
- for o in self.entity.find(self._namedef):
- yield o
- else:
- for objname in self._tagged_host_list():
- arguments = list(self.tags)
- arguments.append(objname)
- yield self.entity(*arguments)
+ for objname in self._tagged_host_list():
+ arguments = list(self.tags)
+ arguments.append(objname)
+ yield self.entity(*arguments)
def _tagged_host_list(self):
cur_dir = self.entity.dir(*self.tags)
@@ -78,20 +73,24 @@
ToolCli.raise_warning()
return retval
- def run_action(self, act, namedef):
- self._action = act
- self._namedef = namedef
+ def run_action(self, unit):
+ self._action, self._namedef = unit
+ return self._run_action()
+
+ def _run_action(self):
for obj in self.host_list():
try:
- a = action.Action(obj, act)
+ a = action.Action(obj, self._action)
msg = a.run()
except action.ActionError as e:
_log.error("Invalid action, reason: %s", str(e))
except BackendError as e:
- _log.error("Error when trying to %s on %s", act, namedef)
+ _log.error("Error when trying to %s on %s", self._action,
+ self._namedef)
_log.error("Failure writing to the kvstore: %s", str(e))
except Exception as e:
- _log.error("Error when trying to %s on %s", act, namedef)
+ _log.error("Error when trying to %s on %s", self._action,
+ self._namedef)
_log.exception("Generic action failure: %s", str(e))
else:
print(msg)
@@ -113,11 +112,95 @@
sys.exit(1)
-def main(cmdline=None):
- if cmdline is None:
- cmdline = list(sys.argv)
- cmdline.pop(0)
+class ToolCliFind(ToolCli):
+ """Subclass used for the --find mode"""
+ def __init__(self, args):
+ self.args = args
+ self.entity = self.object_types[args.object_type]
+ @property
+ def tags(self):
+ return []
+
+ def host_list(self):
+ for o in self.entity.find(self._namedef):
+ yield o
+
+
+class ToolCliByLabel(ToolCli):
+ """Subclass used for the select mode"""
+ def __init__(self, args):
+ self.args = args
+ self.selectors = {}
+ self.parse_selectors()
+ self.entity = self.object_types[args.object_type]
+
+ def parse_selectors(self):
+ for tag in self.args.selector.split(','):
+ k, expr = tag.split('=', 1)
+ self.selectors[k] = re.compile(expr, re.I)
+
+ @property
+ def tags(self):
+ return []
+
+ def get_matching(self, paths, tag):
+ newpaths = []
+ try:
+ regex = self.selectors[tag]
+ except:
+ regex = re.compile('.*')
+
+ for path in paths:
+ for name, _ in KVObject.backend.driver.ls(path):
+ if regex.match(name):
+ newpaths.append(os.path.join(path, name))
+ return newpaths
+
+ def host_list(self):
+ """Gets all the hosts matching our selectors"""
+ places = [self.entity.base_path()]
+ for tag in self.entity._tags:
+ # Find all the paths matching at this level
+ places = self.get_matching(places, tag)
+
+ try:
+ regex = self.selectors['name']
+ except:
+ regex = re.compile('.*')
+
+ objects = []
+ for path in places:
+ for name, entity in KVObject.backend.driver.ls(path):
+ if regex.match(name):
+ objects.append(entity)
+
+ # Any selector that includes multiple objects will show a list of
+ # host that have been selected
+ if self._action != 'get' and len(objects) > 1:
+ self._raise_warning(objects)
+ return objects
+
+ def run_action(self, unit):
+ self._action = unit
+ self._namedef = self.args.selector
+ return self._run_action()
+
+ def raise_warning(self, objects):
+ tag_hosts = defaultdict(list)
+ for obj in objects:
+ dir = os.path.dirname(obj.key).replace(self.entity.base_path(), '')
+ tag_hosts[dir].append(obj.name)
+ print "The selector you chose has selected the following objects:"
+ print json.dumps(tag_hosts)
+ print "Ok to continue? [y/N]"
+ a = raw_input("confctl>")
+ if a.lower() != 'y':
+ print "Aborting"
+ sys.exit(1)
+
+
+def parse_args(cmdline):
parser = argparse.ArgumentParser(
description="Tool to interact with the WMF config store",
epilog="More details at"
@@ -125,29 +208,71 @@
fromfile_prefix_chars='@')
parser.add_argument('--config', help="Optional config file",
default="/etc/conftool/config.yaml")
- parser.add_argument('--tags',
- help="List of comma-separated tags; they need to "
- "match the base tags of the object type you chose.",
- required=False, default=[])
- parser.add_argument('--find', help="Find all instances of the node",
- required=False, default=False, action='store_true')
parser.add_argument('--object-type', dest="object_type",
choices=ToolCli.object_types.keys(), default='node')
- parser.add_argument('--action', action="append", metavar="ACTIONS",
- help="the action to take: "
- " [set/k1=v1:k2=v2...|get|delete]"
- " node|all|re:<regex>|find:node", nargs=2,
- required=True)
parser.add_argument('--debug', action="store_true",
default=False, help="print debug info")
- args = parser.parse_args(cmdline)
+
+ # Subparsers for the three operating models
+ subparsers = parser.add_subparsers(
+ help='Program mode: tags, find or select', dest='mode')
+
+ # Tags mode
+ tags = subparsers.add_parser(
+ 'tags',
+ help="Select a specific service via full list of tags")
+ tags.add_argument(
+ 'taglist',
+ help="List of comma-separated tags; they need to "
+ "match the base tags of the object type you chose.")
+
+ find = subparsers.add_parser('find',
+ help="Find all instances of the node by name")
+ for subparser in [tags, find]:
+ subparser.add_argument('--action', action="append", metavar="ACTIONS",
+ help="the action to take: "
+ " [set/k1=v1:k2=v2...|get|delete]"
+ " node|all|re:<regex>|find:node", nargs=2)
+ select = subparsers.add_parser('select',
+ help="Select nodes via tag selectors")
+ select.add_argument(
+ 'selector',
+ help="Label selector in the form tag=regex: "
+ "dc=eqiad,cluster=cache_.*,service=nginx,name=.*.eqiad.wmnet")
+ select.add_argument('action', action="append", metavar="ACTIONS",
+ help="the action to take: "
+ " [set/k1=v1:k2=v2...|get|delete]")
+
+ return parser.parse_args(cmdline)
+
+
+def mangle_argv():
+ """Basic mangling of the command line arguments"""
+ cmdline = list(sys.argv)
+ cmdline.pop(0)
+ # Backwards compatibility. Ugly but passable
+ for i, arg in enumerate(cmdline):
+ if arg in ['--tags', '--find']:
+ cmdline[i] = arg.replace('--', '')
+ return cmdline
+
+
+def main(cmdline=None):
+ if cmdline is None:
+ cmdline = mangle_argv()
+ args = parse_args(cmdline)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
- cli = ToolCli(args)
+ if args.mode == 'select':
+ cli = ToolCliByLabel(args)
+ elif args.mode == 'find':
+ cli = ToolCliFind(args)
+ else:
+ cli = ToolCli(args)
try:
cli.setup()
@@ -156,8 +281,8 @@
sys.exit(1)
for unit in args.action:
- act, name_def = unit
- cli.run_action(act, name_def)
+ # TODO: fix base class
+ cli.run_action(unit)
if __name__ == '__main__':
diff --git a/conftool/drivers/etcd.py b/conftool/drivers/etcd.py
index e439bfc..8bb10b0 100644
--- a/conftool/drivers/etcd.py
+++ b/conftool/drivers/etcd.py
@@ -68,6 +68,7 @@
@drivers.wrap_exception(etcd.EtcdException)
def ls(self, path):
+ """Given a path, returns a tuple (key, data) for each value found"""
key = self.abspath(path)
try:
res = self.client.read(key)
diff --git a/conftool/tests/integration/test_tool.py
b/conftool/tests/integration/test_tool.py
index f79c526..a078a5b 100644
--- a/conftool/tests/integration/test_tool.py
+++ b/conftool/tests/integration/test_tool.py
@@ -26,7 +26,7 @@
syncer.main(arguments=args)
def generate_args(self, *actions):
- args = ['--tags', "dc=eqiad,cluster=cache_text,service=https"]
+ args = ['tags', "dc=eqiad,cluster=cache_text,service=https"]
for action in actions:
args.append('--action')
args.extend(action.split())
@@ -42,7 +42,7 @@
self.assertEquals(output['cp1008']['pooled'], 'no')
def test_find_node(self):
- args = ['--find', '--action', 'get', 'cp1008']
+ args = ['find', '--action', 'get', 'cp1008']
with captured_output() as (out, err):
tool.main(cmdline=args)
res = out.getvalue().strip()
diff --git a/conftool/tests/unit/test_cli_tool.py
b/conftool/tests/unit/test_cli_tool.py
index 2821879..0e11eec 100644
--- a/conftool/tests/unit/test_cli_tool.py
+++ b/conftool/tests/unit/test_cli_tool.py
@@ -18,49 +18,50 @@
def _mock_args(self, **kw):
arg = mock.MagicMock()
arg.object_type = 'node'
- arg.find = False
+ arg.mode = 'tags'
for prop, value in kw.items():
setattr(arg, prop, value)
return arg
def test_init(self):
"""Test case for `conftool.cli.tool.ToolCli.__init__`"""
- args = self._mock_args(find=False, tags="")
+ args = self._mock_args(taglist="")
with self.assertRaises(SystemExit) as cm:
t = tool.ToolCli(args)
+ t.tags
self.assertEquals(cm.exception.code, 1)
- args = self._mock_args(tags="a=b,b=c,d=2")
+ args = self._mock_args(taglist="a=b,b=c,d=2")
t = tool.ToolCli(args)
- self.assertEquals(t.args.find, False)
+ self.assertEquals(t.args.mode, 'tags')
self.assertItemsEqual(t._tags, ['a=b', 'b=c', 'd=2'])
def test_tags(self):
- args = self._mock_args(tags="dc=a,cluster=b,service=apache2")
+ args = self._mock_args(taglist="dc=a,cluster=b,service=apache2")
t = tool.ToolCli(args)
self.assertItemsEqual(t.tags, ['a', 'b', 'apache2'])
- args = self._mock_args(find=True)
- t = tool.ToolCli(args)
+ args = self._mock_args(mode='find')
+ t = tool.ToolCliFind(args)
self.assertEquals(t.tags, [])
args = self._mock_args(
- find=True,
+ mode='find',
object_type='node',
)
- t = tool.ToolCli(args)
+ t = tool.ToolCliFind(args)
self.assertEquals(t.entity.__name__, 'Node')
def test_host_list_find(self):
args = self._mock_args(
- find=True,
+ mode="find",
object_type='node',
)
- t = tool.ToolCli(args)
+ t = tool.ToolCliFind(args)
t._namedef = 'cp1048.example.com'
res = [
- node.Node('mycloset', 'cache_unicorns',
+ node.Node('test', 'cache',
'ssl', 'cp1048.example.com'),
- node.Node('mycloset', 'cache_unicorns',
+ node.Node('test', 'cache',
'http', 'cp1048.example.com'),
]
t.entity.find = mock.MagicMock(return_value=res)
@@ -76,7 +77,7 @@
('cp1014.local', {'pooled': 'no'})
]
self._mock_list(host_dir)
- args = self._mock_args(tags="dc=a,cluster=b,service=apache2")
+ args = self._mock_args(taglist="dc=a,cluster=b,service=apache2")
def tagged(args, name, act):
t = tool.ToolCli(args)
@@ -107,3 +108,11 @@
# Majority of nodes via a regex will raise a system exit
with self.assertRaises(SystemExit):
tagged(args, 're:cp10(11|20)\.example\.com', 'set')
+
+ def test_parse_args(self):
+ # Taglist
+ cmdline = ['tags', 'dc=a,cluster=b', '--action', 'get', 'all']
+ args = tool.parse_args(cmdline)
+ self.assertEquals(args.mode, 'tags')
+ self.assertEquals(args.taglist, 'dc=a,cluster=b')
+ self.assertEquals(args.action, [['get', 'all']])
--
To view, visit https://gerrit.wikimedia.org/r/278552
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2ee04cd3e8eedf8d3f985b85099932f1d48d175b
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/conftool
Gerrit-Branch: master
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
