jenkins-bot has submitted this change and it was merged.
Change subject: Add pattern-matching arg to limit deploy hosts
......................................................................
Add pattern-matching arg to limit deploy hosts
Adds a flag to the deploy command `--limit-hosts` or `-l` that accepts a
pattern in one of the following formats:
- `~[regex]` - if the pattern starts with ~ it is interpreted as a
regular expression, this is not combined with any other option
- `![pattern]` - can be combined with any other pattern to negate
that pattern. May be combined with other pattern matching options.
- `host[01:10]` - range matching, works for ascii chars and numbers,
including numbers with a leading 0, may be combined with other
pattern-matching options.
- `host*` - Matches 0 or more characters in the set A-z, '_', '.', or
'-'. May be combined with other pattern matching options.
This pattern is applied to the `dsh_targets` file to return a sub-set of
hosts to use as a deployment target.
Change-Id: I3ec5574c987ac576acc6cf23958542480a63ae66
---
M scap/main.py
M scap/utils.py
M tests/utils_test.py
3 files changed, 137 insertions(+), 2 deletions(-)
Approvals:
Thcipriani: Looks good to me, approved
20after4: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/scap/main.py b/scap/main.py
index fe03aee..c12d64c 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -616,6 +616,8 @@
"""
@cli.argument('-r', '--rev', default='HEAD', help='Revision to deploy')
+ @cli.argument('-l', '--limit-hosts', default='all',
+ help='Limit deploy to hosts matching expression')
def main(self, *extra_args):
logger = self.get_logger()
repo = self.config['git_repo']
@@ -633,7 +635,16 @@
'Script must be run from deployment repository under {}'
.format(deploy_dir))
- targets = utils.read_dsh_hosts_file(self.config['dsh_targets'])
+ targets = utils.get_target_hosts(
+ self.arguments.limit_hosts,
+ utils.read_dsh_hosts_file(self.config['dsh_targets'])
+ )
+
+ logger.info(
+ 'Deploy will run on the following targets: \n\t- {}'.format(
+ '\n\t- '.join(targets)
+ )
+ )
# batch_size not required, don't allow a batch_size > 80 if set
batch_size = self.config.get('batch_size', 80)
diff --git a/scap/utils.py b/scap/utils.py
index 9a65569..78e3746 100644
--- a/scap/utils.py
+++ b/scap/utils.py
@@ -9,6 +9,7 @@
import errno
import fcntl
import hashlib
+import inspect
import json
import logging
import os
@@ -16,10 +17,10 @@
import random
import re
import socket
+import string
import struct
import subprocess
import tempfile
-import inspect
from . import ansi
from datetime import datetime
@@ -560,3 +561,75 @@
}
return (tag, tag_info)
+
+
+def get_target_hosts(pattern, hosts):
+ """Returns a subset of hosts based on wildcards
+
+ if the pattern can specify a range of the format '[start:end]'
+
+ if the supplied pattern begins with ``~`` then it is treated as a
+ regular expression.
+
+ If the pattern begins with ``!`` then it is negated.
+ """
+ # Return early if there's no special pattern
+ if pattern == '*' or pattern == 'all':
+ return hosts
+
+ # If pattern is a regex, handle that and return
+ if pattern[0] == '~':
+ regex = re.compile(pattern[1:])
+ return [target for target in hosts if regex.match(target)]
+
+ # Handle replacements of anything like [*:*] in pattern
+ has_range = lambda x: 0 <= x.find('[') < x.find(':') < x.find(']')
+
+ patterns = []
+ rpattern = pattern
+ while(has_range(rpattern)):
+ head, nrange, tail = rpattern.replace(
+ '[', '|', 1).replace(']', '|', 1).split('|')
+
+ beg, end = nrange.split(':')
+ zfill = len(end) if (len(beg) > 0 and beg.startswith('0')) else 0
+
+ if (zfill != 0 and len(beg) != len(end)) or beg > end:
+ raise ValueError("Host range incorrectly specified")
+
+ try:
+ asc = string.ascii_letters
+ seq = asc[asc.index(beg):asc.index(end) + 1]
+ except ValueError: # numeric range
+ seq = range(int(beg), int(end) + 1)
+
+ patterns = [''.join([head, str(i).zfill(zfill), tail]) for i in seq]
+ rpattern = rpattern[rpattern.find(']') + 1:]
+
+ # If there weren't range replacements, make pattern an array
+ if len(patterns) == 0:
+ patterns = [pattern]
+
+ targets = []
+ for pattern in patterns:
+ # remove any leading '!'
+ test_pattern = pattern.lstrip('!')
+
+ # change '.' to literal period
+ test_pattern = test_pattern.replace('.', '\.')
+
+ # convert '*' to match a-Z, 0-9, _, -, or .
+ test_pattern = test_pattern.replace('*', '[\w\.-]*')
+
+ # Add beginning and end marks
+ test_pattern = '^{}$'.format(test_pattern)
+
+ regex = re.compile(test_pattern)
+
+ targets.extend([host for host in hosts if regex.match(host)])
+
+ # handle regation of patterns by inverting
+ if pattern.startswith('!'):
+ targets = list(set(targets) ^ set(hosts))
+
+ return targets
diff --git a/tests/utils_test.py b/tests/utils_test.py
index 16e36fc..f4c0f32 100644
--- a/tests/utils_test.py
+++ b/tests/utils_test.py
@@ -37,3 +37,54 @@
utils.check_valid_json_file,
path
)
+
+ def test_check_target_hosts(self):
+ hosts = [
+ 'test01',
+ 'test02',
+ 'test01.eqiad.wmnet',
+ 'test02.eqiad.wmnet',
+ 'test01.staging.eqiad.wmflabs',
+ 'test02.staging.eqiad.wmflabs'
+ ]
+
+ gth = lambda x: sorted(utils.get_target_hosts(x, hosts))
+
+ tests = [(gth('*'), sorted(hosts)),
+ (gth('all'), sorted(hosts)),
+ (gth('test01FeqiadFwmnet'), []),
+ (gth('test01.eqiad.wmnet'),
+ ['test01.eqiad.wmnet']),
+ (gth('test[01:02].eqiad.*'),
+ ['test01.eqiad.wmnet',
+ 'test02.eqiad.wmnet']),
+ (gth('test*.eqiad.*'),
+ ['test01.eqiad.wmnet',
+ 'test01.staging.eqiad.wmflabs',
+ 'test02.eqiad.wmnet',
+ 'test02.staging.eqiad.wmflabs']),
+ (gth('!test[01:02].eqiad.*'),
+ ['test01',
+ 'test01.staging.eqiad.wmflabs',
+ 'test02',
+ 'test02.staging.eqiad.wmflabs']),
+ (gth('~test.*.eqiad.wm(net|flabs)'),
+ ['test01.eqiad.wmnet',
+ 'test01.staging.eqiad.wmflabs',
+ 'test02.eqiad.wmnet',
+ 'test02.staging.eqiad.wmflabs']),
+ (gth('test*.eqia[a:d].wmflabs'),
+ ['test01.staging.eqiad.wmflabs',
+ 'test02.staging.eqiad.wmflabs']),
+ (gth('test[01:02]'), ['test01', 'test02']),
+ (gth('test02'), ['test02']),
+ (gth('tes?02'), []),
+ (gth('tes.02'), []),
+ ]
+
+ for test in tests:
+ self.assertEqual(test[0], test[1])
+
+ with self.assertRaises(ValueError):
+ gth('test[10:01]')
+ gth('test[z:a]')
--
To view, visit https://gerrit.wikimedia.org/r/238208
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3ec5574c987ac576acc6cf23958542480a63ae66
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <tcipri...@wikimedia.org>
Gerrit-Reviewer: 20after4 <mmod...@wikimedia.org>
Gerrit-Reviewer: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: Chad <ch...@wikimedia.org>
Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Thcipriani <tcipri...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
