Chad has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/355635 )
Change subject: Scap clean: Rewrite to just do stuff on masters then sync
......................................................................
Scap clean: Rewrite to just do stuff on masters then sync
It's what I should've done from the get go. I'm so stupid
Change-Id: I7cf2e4703d5c9b71879a46e394a08fa3ff42fbfd
---
M scap/plugins/clean.py
1 file changed, 20 insertions(+), 40 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/35/355635/1
diff --git a/scap/plugins/clean.py b/scap/plugins/clean.py
index 0ef8714..1179d6c 100644
--- a/scap/plugins/clean.py
+++ b/scap/plugins/clean.py
@@ -11,7 +11,6 @@
import scap.git as git
import scap.log as log
import scap.main as main
-import scap.ssh as ssh
import scap.utils as utils
DELETABLE_TYPES = [
@@ -57,6 +56,10 @@
help='Delete everything (not just static assets).')
def main(self, *extra_args):
""" Clean old branches from the cluster for space savings! """
+ return super(Clean, self).main(*extra_args)
+
+ def _before_cluster_sync(self):
+ """Before we sync to the cluster, do our cleanup"""
if self.arguments.branch in self.active_wikiversions().keys():
raise ValueError('Branch "%s" is still in use, aborting' %
self.arguments.branch)
@@ -74,7 +77,6 @@
(4.2) Then targets
"""
stage_dir = os.path.join(self.config['stage_dir'], 'php-%s' % branch)
- deploy_dir = os.path.join(self.config['deploy_dir'], 'php-%s' % branch)
if delete:
gerrit_prune_cmd = ['git', 'push', 'origin', '--quiet', '--delete',
@@ -95,50 +97,28 @@
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info('Failed to prune core branch')
- # Prune cache junk from masters used by l10nupdate
- self.execute_remote(
- 'clean-masters-l10nupdate-cache',
- self._get_master_list(),
- ['sudo', '-u', 'www-data', 'rm', '-fR',
- '/var/lib/l10nupdate/caches/cache-%s' % branch]
- )
+ commands = {
+ 'clean-l10nupdate-cache':
+ ['sudo', '-u', 'www-data', 'rm', '-fR',
+ '/var/lib/l10nupdate/caches/cache-%s' % branch],
+ 'clean-l10nupdate-owned-files':
+ ['sudo', '-u', 'l10nupdate', 'find', stage_dir,
+ '-user', 'l10nupdate', '-delete'],
+ 'cleaning-branch':
+ ' '.join(clean_command(stage_dir, delete))
+ }
- # Prune junk from masters owned by l10nupdate
- self.execute_remote(
- 'clean-masters-l10nupdate',
- self._get_master_list(),
- ['sudo', '-u', 'l10nupdate', 'find', stage_dir,
- '-user', 'l10nupdate', '-delete']
- )
+ for name, command in commands:
+ with log.Timer(name, self.get_stats()):
+ subprocess.check_call(command)
- # Update active master (passive gets it on next sync)
- master_command = ' '.join(clean_command(stage_dir, delete))
- subprocess.check_call(master_command, shell=True)
-
- # Update apaches
- self.execute_remote(
- 'clean-apaches',
- self._get_target_list(),
- clean_command(deploy_dir, delete)
- )
-
- announce = 'Pruned MediaWiki: %s' % branch
- if not delete:
+ def _after_lock_release(self):
+ announce = 'Pruned MediaWiki: %s' % self.arguments.branch
+ if not self.arguments.delete:
announce += ' [keeping static files]'
self.announce(announce + ' (duration: %s)' %
utils.human_duration(self.get_duration()))
-
- def execute_remote(self, description, targets, command):
- with log.Timer(description, self.get_stats()):
- clean_job = ssh.Job(targets, user=self.config['ssh_user'])
- clean_job.shuffle()
- clean_job.command(command)
- clean_job.progress(
- log.reporter(description, self.config['fancy_progress']))
- succeeded, failed = clean_job.run()
- if failed:
- self.get_logger().warning('%d had clean errors', failed)
def clean_command(path, delete):
--
To view, visit https://gerrit.wikimedia.org/r/355635
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7cf2e4703d5c9b71879a46e394a08fa3ff42fbfd
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Chad <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
