Thcipriani has uploaded a new change for review. https://gerrit.wikimedia.org/r/238208
Change subject: Add pattern-matching arg to limit deploy hosts ...................................................................... Add pattern-matching arg to limit deploy hosts Adds a flag to the deploy command `--limit-hosts` or `-l` that accepts a pattern in one of the following formats: - `~[regex]` - if the pattern starts with ~ it is interpreted as a regular expression, this is not combined with any other option - `![pattern]` - can be combined with any other pattern to negate that pattern. May be combined with other pattern matching options. - `host[01:10]` - range matching, works for ascii chars and numbers, including numbers with a leading 0, may be combined with other pattern-matching options. - `host*` - Matches 0 or more characters in the set A-z, '_', '.', or '-'. May be combined with other pattern matching options. This pattern is applied to the `dsh_targets` file to return a sub-set of hosts to use as a deployment target. Change-Id: I3ec5574c987ac576acc6cf23958542480a63ae66 --- M scap/main.py M scap/utils.py M tests/utils_test.py 3 files changed, 137 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap refs/changes/08/238208/1 diff --git a/scap/main.py b/scap/main.py index fe03aee..c12d64c 100644 --- a/scap/main.py +++ b/scap/main.py @@ -616,6 +616,8 @@ """ @cli.argument('-r', '--rev', default='HEAD', help='Revision to deploy') + @cli.argument('-l', '--limit-hosts', default='all', + help='Limit deploy to hosts matching expression') def main(self, *extra_args): logger = self.get_logger() repo = self.config['git_repo'] @@ -633,7 +635,16 @@ 'Script must be run from deployment repository under {}' .format(deploy_dir)) - targets = utils.read_dsh_hosts_file(self.config['dsh_targets']) + targets = utils.get_target_hosts( + self.arguments.limit_hosts, + utils.read_dsh_hosts_file(self.config['dsh_targets']) + ) + + logger.info( + 'Deploy will run on the following targets: \n\t- {}'.format( + '\n\t- '.join(targets) + ) + ) # batch_size not required, don't allow a batch_size > 80 if set batch_size = self.config.get('batch_size', 80) diff --git a/scap/utils.py b/scap/utils.py index 9a65569..78e3746 100644 --- a/scap/utils.py +++ b/scap/utils.py @@ -9,6 +9,7 @@ import errno import fcntl import hashlib +import inspect import json import logging import os @@ -16,10 +17,10 @@ import random import re import socket +import string import struct import subprocess import tempfile -import inspect from . import ansi from datetime import datetime @@ -560,3 +561,75 @@ } return (tag, tag_info) + + +def get_target_hosts(pattern, hosts): + """Returns a subset of hosts based on wildcards + + if the pattern can specify a range of the format '[start:end]' + + if the supplied pattern begins with ``~`` then it is treated as a + regular expression. + + If the pattern begins with ``!`` then it is negated. + """ + # Return early if there's no special pattern + if pattern == '*' or pattern == 'all': + return hosts + + # If pattern is a regex, handle that and return + if pattern[0] == '~': + regex = re.compile(pattern[1:]) + return [target for target in hosts if regex.match(target)] + + # Handle replacements of anything like [*:*] in pattern + has_range = lambda x: 0 <= x.find('[') < x.find(':') < x.find(']') + + patterns = [] + rpattern = pattern + while(has_range(rpattern)): + head, nrange, tail = rpattern.replace( + '[', '|', 1).replace(']', '|', 1).split('|') + + beg, end = nrange.split(':') + zfill = len(end) if (len(beg) > 0 and beg.startswith('0')) else 0 + + if (zfill != 0 and len(beg) != len(end)) or beg > end: + raise ValueError("Host range incorrectly specified") + + try: + asc = string.ascii_letters + seq = asc[asc.index(beg):asc.index(end) + 1] + except ValueError: # numeric range + seq = range(int(beg), int(end) + 1) + + patterns = [''.join([head, str(i).zfill(zfill), tail]) for i in seq] + rpattern = rpattern[rpattern.find(']') + 1:] + + # If there weren't range replacements, make pattern an array + if len(patterns) == 0: + patterns = [pattern] + + targets = [] + for pattern in patterns: + # remove any leading '!' + test_pattern = pattern.lstrip('!') + + # change '.' to literal period + test_pattern = test_pattern.replace('.', '\.') + + # convert '*' to match a-Z, 0-9, _, -, or . + test_pattern = test_pattern.replace('*', '[\w\.-]*') + + # Add beginning and end marks + test_pattern = '^{}$'.format(test_pattern) + + regex = re.compile(test_pattern) + + targets.extend([host for host in hosts if regex.match(host)]) + + # handle regation of patterns by inverting + if pattern.startswith('!'): + targets = list(set(targets) ^ set(hosts)) + + return targets diff --git a/tests/utils_test.py b/tests/utils_test.py index 16e36fc..f4c0f32 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -37,3 +37,54 @@ utils.check_valid_json_file, path ) + + def test_check_target_hosts(self): + hosts = [ + 'test01', + 'test02', + 'test01.eqiad.wmnet', + 'test02.eqiad.wmnet', + 'test01.staging.eqiad.wmflabs', + 'test02.staging.eqiad.wmflabs' + ] + + gth = lambda x: sorted(utils.get_target_hosts(x, hosts)) + + tests = [(gth('*'), sorted(hosts)), + (gth('all'), sorted(hosts)), + (gth('test01FeqiadFwmnet'), []), + (gth('test01.eqiad.wmnet'), + ['test01.eqiad.wmnet']), + (gth('test[01:02].eqiad.*'), + ['test01.eqiad.wmnet', + 'test02.eqiad.wmnet']), + (gth('test*.eqiad.*'), + ['test01.eqiad.wmnet', + 'test01.staging.eqiad.wmflabs', + 'test02.eqiad.wmnet', + 'test02.staging.eqiad.wmflabs']), + (gth('!test[01:02].eqiad.*'), + ['test01', + 'test01.staging.eqiad.wmflabs', + 'test02', + 'test02.staging.eqiad.wmflabs']), + (gth('~test.*.eqiad.wm(net|flabs)'), + ['test01.eqiad.wmnet', + 'test01.staging.eqiad.wmflabs', + 'test02.eqiad.wmnet', + 'test02.staging.eqiad.wmflabs']), + (gth('test*.eqia[a:d].wmflabs'), + ['test01.staging.eqiad.wmflabs', + 'test02.staging.eqiad.wmflabs']), + (gth('test[01:02]'), ['test01', 'test02']), + (gth('test02'), ['test02']), + (gth('tes?02'), []), + (gth('tes.02'), []), + ] + + for test in tests: + self.assertEqual(test[0], test[1]) + + with self.assertRaises(ValueError): + gth('test[10:01]') + gth('test[z:a]') -- To view, visit https://gerrit.wikimedia.org/r/238208 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3ec5574c987ac576acc6cf23958542480a63ae66 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/tools/scap Gerrit-Branch: master Gerrit-Owner: Thcipriani <tcipri...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits