jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/355635 )
Change subject: Scap clean: Rewrite to just do stuff on masters then sync
......................................................................
Scap clean: Rewrite to just do stuff on masters then sync
It's what I should've done from the get go. I'm so stupid
Change-Id: I7cf2e4703d5c9b71879a46e394a08fa3ff42fbfd
---
M scap/plugins/clean.py
1 file changed, 26 insertions(+), 51 deletions(-)
Approvals:
Chad: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scap/plugins/clean.py b/scap/plugins/clean.py
index 0ef8714..9edfa42 100644
--- a/scap/plugins/clean.py
+++ b/scap/plugins/clean.py
@@ -11,7 +11,6 @@
import scap.git as git
import scap.log as log
import scap.main as main
-import scap.ssh as ssh
import scap.utils as utils
DELETABLE_TYPES = [
@@ -57,6 +56,9 @@
help='Delete everything (not just static assets).')
def main(self, *extra_args):
""" Clean old branches from the cluster for space savings! """
+ return super(Clean, self).main(*extra_args)
+
+ def _before_cluster_sync(self):
if self.arguments.branch in self.active_wikiversions().keys():
raise ValueError('Branch "%s" is still in use, aborting' %
self.arguments.branch)
@@ -74,7 +76,17 @@
(4.2) Then targets
"""
stage_dir = os.path.join(self.config['stage_dir'], 'php-%s' % branch)
- deploy_dir = os.path.join(self.config['deploy_dir'], 'php-%s' % branch)
+ if not os.path.isdir(stage_dir):
+ raise ValueError('No such branch exists, aborting')
+
+ commands = {
+ 'clean-l10nupdate-cache':
+ ['sudo', '-u', 'www-data', 'rm', '-fR',
+ '/var/lib/l10nupdate/caches/cache-%s' % branch],
+ 'clean-l10nupdate-owned-files':
+ ['sudo', '-u', 'l10nupdate', 'find', stage_dir,
+ '-user', 'l10nupdate', '-delete']
+ }
if delete:
gerrit_prune_cmd = ['git', 'push', 'origin', '--quiet', '--delete',
@@ -94,58 +106,21 @@
with utils.cd(stage_dir):
if subprocess.call(gerrit_prune_cmd) != 0:
logger.info('Failed to prune core branch')
+ commands['cleaning-branch'] = ['rm', '-fR', stage_dir]
+ else:
+ regex = r'".*\.?({0})$"'.format('|'.join(DELETABLE_TYPES))
+ commands['cleaning-branch'] = ['find', stage_dir, '-type', 'f',
+ '-regextype', 'posix-extended',
+ '-regex', regex, '-delete']
- # Prune cache junk from masters used by l10nupdate
- self.execute_remote(
- 'clean-masters-l10nupdate-cache',
- self._get_master_list(),
- ['sudo', '-u', 'www-data', 'rm', '-fR',
- '/var/lib/l10nupdate/caches/cache-%s' % branch]
- )
+ for name, command in commands.iteritems():
+ with log.Timer(name + '-' + branch, self.get_stats()):
+ subprocess.check_call(command)
- # Prune junk from masters owned by l10nupdate
- self.execute_remote(
- 'clean-masters-l10nupdate',
- self._get_master_list(),
- ['sudo', '-u', 'l10nupdate', 'find', stage_dir,
- '-user', 'l10nupdate', '-delete']
- )
-
- # Update active master (passive gets it on next sync)
- master_command = ' '.join(clean_command(stage_dir, delete))
- subprocess.check_call(master_command, shell=True)
-
- # Update apaches
- self.execute_remote(
- 'clean-apaches',
- self._get_target_list(),
- clean_command(deploy_dir, delete)
- )
-
- announce = 'Pruned MediaWiki: %s' % branch
- if not delete:
+ def _after_lock_release(self):
+ announce = 'Pruned MediaWiki: %s' % self.arguments.branch
+ if not self.arguments.delete:
announce += ' [keeping static files]'
self.announce(announce + ' (duration: %s)' %
utils.human_duration(self.get_duration()))
-
- def execute_remote(self, description, targets, command):
- with log.Timer(description, self.get_stats()):
- clean_job = ssh.Job(targets, user=self.config['ssh_user'])
- clean_job.shuffle()
- clean_job.command(command)
- clean_job.progress(
- log.reporter(description, self.config['fancy_progress']))
- succeeded, failed = clean_job.run()
- if failed:
- self.get_logger().warning('%d had clean errors', failed)
-
-
-def clean_command(path, delete):
- """Generate a command depending on where we are and what we're doing"""
- regex = '".*\.?(%s)$"' % ('|'.join(DELETABLE_TYPES))
- if delete:
- return ['rm', '-fR', path]
- else:
- return ['find', path, '-type', 'f', '-regextype', 'posix-extended',
- '-regex', regex, '-delete']
--
To view, visit https://gerrit.wikimedia.org/r/355635
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7cf2e4703d5c9b71879a46e394a08fa3ff42fbfd
Gerrit-PatchSet: 3
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Chad <[email protected]>
Gerrit-Reviewer: Chad <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
