BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/155677
Change subject: Make sync-common update l10n cdb files by default
......................................................................
Make sync-common update l10n cdb files by default
Deployers (and puppet) mostly assume that sync-common is all that is
needed to update a host with the deploy master. Nobody remembers that
you now have to run scap-rebuild-cdbs too. Fix this by making the
default behavior of sync-common include updating cdbs from the synced
json files. `scap` and friends will call as
`sync-common --no-update-l10n ...` to preserve existing split workflow.
Bug: 69792
Change-Id: I634fe0a19b909cfc84925e4661ee2bc320359095
---
M scap/log.py
M scap/main.py
M scap/tasks.py
3 files changed, 32 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/77/155677/1
diff --git a/scap/log.py b/scap/log.py
index c4111cb..5547f30 100644
--- a/scap/log.py
+++ b/scap/log.py
@@ -232,6 +232,18 @@
self.ok, self.failed, self.remaining)
+class MuteReporter(ProgressReporter):
+ """A report that declines to report anything."""
+ def __init__(self, name='', expect=0, fd=sys.stderr):
+ super(self.__class__, self).__init__(name)
+
+ def _progress(self):
+ pass
+
+ def finish(self):
+ pass
+
+
class Stats(object):
"""A simple StatsD metric client that can log measurements and counts to
a remote StatsD host.
diff --git a/scap/main.py b/scap/main.py
index 723d71b..4299d9a 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -80,7 +80,7 @@
def _proxy_sync_command(self):
"""Synchronization command to run on the proxy hosts."""
- cmd = ['sync-common']
+ cmd = ['sync-common', '--no-update-l10n']
if self.verbose:
cmd.append('--verbose')
return cmd
@@ -161,6 +161,8 @@
class RebuildCdbs(cli.Application):
"""Rebuild localization cache CDB files from the JSON versions."""
+ @cli.argument('--no-progress', action='store_true', dest='mute',
+ help='Do not show progress indicator.')
def main(self, *extra_args):
self._run_as('mwdeploy')
self._assert_current_user('mwdeploy')
@@ -172,7 +174,8 @@
for version, wikidb in self.active_wikiversions().items():
cache_dir = os.path.join(self.config['deploy_dir'],
'php-%s' % version, 'cache', 'l10n')
- tasks.merge_cdb_updates(cache_dir, use_cores, True)
+ tasks.merge_cdb_updates(
+ cache_dir, use_cores, True, self.arguments.mute)
class Scap(AbstractSync):
@@ -264,6 +267,8 @@
class SyncCommon(cli.Application):
"""Sync local MediaWiki deployment directory with deploy server state."""
+ @cli.argument('--no-update-l10n', action='store_false', dest='update_l10n',
+ help='Do not update l10n cache files.')
@cli.argument('-i', '--include', default=None, action='append',
help='Rsync include pattern to limit transfer to.'
'End directories with a trailing `/***`. Can be used multiple times.')
@@ -276,6 +281,9 @@
sync_from=self.arguments.servers,
verbose=self.verbose
)
+ if self.arguments.update_l10n:
+ utils.sudo_check_call('mwdeploy',
+ 'scap-rebuild-cdbs --no-progress', self.get_logger())
return 0
@@ -283,7 +291,7 @@
"""Sync dblist files to the cluster."""
def _proxy_sync_command(self):
- cmd = ['sync-common', '--include', '*.dblist']
+ cmd = ['sync-common', '--no-update-l10n', '--include', '*.dblist']
if self.verbose:
cmd.append('--verbose')
return cmd
@@ -315,7 +323,7 @@
tasks.check_php_syntax(abspath)
def _proxy_sync_command(self):
- cmd = ['sync-common']
+ cmd = ['sync-common', '--no-update-l10n']
if '/' in self.include:
parts = self.include.split('/')
@@ -342,7 +350,7 @@
def _proxy_sync_command(self):
cmd = [
- 'sync-common',
+ 'sync-common', '--no-update-l10n',
'--include', 'docroot/***',
'--include', 'w/***',
]
@@ -376,7 +384,7 @@
subprocess.check_call('/usr/bin/php -l %s' % abspath, shell=True)
def _proxy_sync_command(self):
- cmd = ['sync-common']
+ cmd = ['sync-common', '--no-update-l10n']
if '/' in self.include:
parts = self.include.split('/')
diff --git a/scap/tasks.py b/scap/tasks.py
index cd198dd..b47979d 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -149,12 +149,13 @@
logger.info('Compiled %s to %s', json_file, cdb_file)
-def merge_cdb_updates(directory, pool_size, trust_mtime=False):
+def merge_cdb_updates(directory, pool_size, trust_mtime=False, mute=False):
"""Update l10n CDB files using JSON data.
:param directory: L10n cache directory
:param pool_size: Number of parallel processes to use
:param trust_mtime: Trust file modification time?
+ :param mute: Disable progress indicator
"""
logger = logging.getLogger('merge_cdb_updates')
@@ -170,7 +171,10 @@
pool = multiprocessing.Pool(pool_size)
updated = 0
- reporter = log.ProgressReporter('l10n merge')
+ if mute:
+ reporter = log.MuteReporter()
+ else:
+ reporter = log.ProgressReporter('l10n merge')
reporter.expect(len(files))
reporter.start()
--
To view, visit https://gerrit.wikimedia.org/r/155677
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I634fe0a19b909cfc84925e4661ee2bc320359095
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
