Ori.livneh has uploaded a new change for review. https://gerrit.wikimedia.org/r/174592
Change subject: Make the SSH user and authentication socket configurable ...................................................................... Make the SSH user and authentication socket configurable * Add a config key, 'ssh_user', which defaults to the current username. * Add a config key, 'ssh_auth_sock', which is unset by default. If set, it will override SSH_AUTH_SOCK in the environment. This is done before _assert_auth_sock is called. * Pass ssh_user to Job and from there to cluster_ssh. * ssh_auth_sock doesn't need to be passed around. If it is set, it is set in the environment, which every child process automatically inherits. Change-Id: I5782e8cbe24bbd28d38967bc16f5f44721dc646f --- M scap.cfg M scap/cli.py M scap/config.py M scap/main.py M scap/ssh.py M scap/tasks.py 6 files changed, 34 insertions(+), 10 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap refs/changes/92/174592/1 diff --git a/scap.cfg b/scap.cfg index 5a285fe..e3a7a68 100644 --- a/scap.cfg +++ b/scap.cfg @@ -72,6 +72,8 @@ statsd_host: statsd.eqiad.wmnet tcpircbot_host: neon.wikimedia.org udp2log_host: fluorine.eqiad.wmnet +ssh_auth_sock: /run/keyholder/proxy.sock +ssh_user: mwdeploy [eqiad.wmflabs] diff --git a/scap/cli.py b/scap/cli.py index 52bd532..a02b366 100644 --- a/scap/cli.py +++ b/scap/cli.py @@ -168,6 +168,12 @@ """Setup logging.""" log.setup_loggers(self.config, self.arguments.loglevel) + def _setup_environ(self): + """Setup shell environment.""" + auth_sock = self.config.get('ssh_auth_sock') + if auth_sock is not None: + os.environ['SSH_AUTH_SOCK'] = auth_sock + def main(self, *extra_args): """Main business logic of the application. @@ -263,6 +269,7 @@ app.arguments = args app._load_config() app._setup_loggers() + app._setup_environ() exit_status = app.main(extra_args) except SystemExit as ex: diff --git a/scap/config.py b/scap/config.py index f5e5028..b3c8766 100644 --- a/scap/config.py +++ b/scap/config.py @@ -6,6 +6,7 @@ """ import ConfigParser +import getpass import os import socket @@ -23,6 +24,7 @@ 'udp2log_host': None, 'udp2log_port': '8420', 'wmf_realm': 'production', + 'ssh_user': getpass.getuser(), 'datacenter': 'pmtpa', } diff --git a/scap/main.py b/scap/main.py index 72c589c..63fc0fa 100644 --- a/scap/main.py +++ b/scap/main.py @@ -40,7 +40,7 @@ # Update proxies proxies = self._get_proxy_list() with log.Timer('sync-proxies', self.get_stats()): - update_proxies = ssh.Job(proxies) + update_proxies = ssh.Job(proxies, user=self.config['ssh_user']) update_proxies.command(self._proxy_sync_command()) update_proxies.progress('sync-proxies') succeeded, failed = update_proxies.run() @@ -51,7 +51,8 @@ # Update apaches with log.Timer('sync-apaches', self.get_stats()): - update_apaches = ssh.Job(self._get_apache_list()) + update_apaches = ssh.Job(self._get_apache_list(), + user=self.config['ssh_user']) update_apaches.exclude_hosts(proxies) update_apaches.shuffle() update_apaches.command(self._apache_sync_command(proxies)) @@ -223,7 +224,8 @@ def _after_cluster_sync(self): # Ask apaches to rebuild l10n CDB files with log.Timer('scap-rebuild-cdbs', self.get_stats()): - rebuild_cdbs = ssh.Job(self._get_apache_list()) + rebuild_cdbs = ssh.Job(self._get_apache_list(), + user=self.config['ssh_user']) rebuild_cdbs.shuffle() rebuild_cdbs.command('sudo -u mwdeploy -n -- %s' % self.get_script_path('scap-rebuild-cdbs')) diff --git a/scap/ssh.py b/scap/ssh.py index 11788c1..a8389a2 100644 --- a/scap/ssh.py +++ b/scap/ssh.py @@ -24,10 +24,11 @@ """Execute a job on a group of remote hosts via ssh.""" _logger = None - def __init__(self, hosts=None, command=None): + def __init__(self, hosts=None, command=None, user=None): self.hosts(hosts or []) self._command = command self._reporter = None + self._user = user def get_logger(self): """Lazy getter for a logger instance.""" @@ -88,7 +89,8 @@ if self._reporter: return self._run_with_reporter(batch_size) else: - return list(cluster_ssh(self._hosts, self._command, batch_size)) + return list(cluster_ssh(self._hosts, self._command, self._user, + batch_size)) def _run_with_reporter(self, batch_size): """Run job and feed results to a :class:`log.ProgressReporter` as they @@ -97,7 +99,7 @@ self._reporter.start() for host, status, output in cluster_ssh( - self._hosts, self._command, batch_size): + self._hosts, self._command, self._user, batch_size): if status == 0: self._reporter.add_success() else: @@ -108,7 +110,7 @@ return self._reporter.ok, self._reporter.failed -def cluster_ssh(hosts, command, limit=80): +def cluster_ssh(hosts, command, user=None, limit=80): """Run a command via SSH on multiple hosts concurrently.""" hosts = set(hosts) @@ -124,7 +126,11 @@ while hosts or procs: if hosts and len(procs) < limit: host = hosts.pop() - ssh_command = SSH + (host,) + tuple(command) + ssh_command = list(SSH) + if user: + ssh_command.append('-l%s' % user) + ssh_command.append(host) + ssh_command.extend(command) proc = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) procs[proc.pid] = (proc, host) diff --git a/scap/tasks.py b/scap/tasks.py index 71769cc..56c1187 100644 --- a/scap/tasks.py +++ b/scap/tasks.py @@ -226,8 +226,13 @@ # Purge from deploy directroy across cluster # --force option given to rm to ignore missing files as before +<<<<<<< HEAD purge = ssh.Job().role('mediawiki-installation') purge.command('sudo -u mwdeploy -n -- /bin/rm ' +======= + purge = ssh.Job(user=cfg['ssh_user']).role('mediawiki-installation') + purge.command('sudo -u mwdeploy /bin/rm ' +>>>>>>> Make the SSH user and authentication socket configurable '--recursive --force %s/*' % deployed_l10n) purge.progress('l10n purge').run() @@ -301,8 +306,8 @@ with log.Timer('sync_wikiversions', stats): compile_wikiversions_cdb('stage', cfg) - rsync = ssh.Job(hosts).shuffle() - rsync.command('sudo -u mwdeploy -n -- /usr/bin/rsync -l ' + rsync = ssh.Job(hosts, user=cfg['ssh_username']).shuffle() + rsync.command('sudo -u mwdeploy /usr/bin/rsync -l ' '%(master_rsync)s::common/wikiversions*.{json,cdb} ' '%(deploy_dir)s' % cfg) return rsync.progress('sync_wikiversions').run() -- To view, visit https://gerrit.wikimedia.org/r/174592 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5782e8cbe24bbd28d38967bc16f5f44721dc646f Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/tools/scap Gerrit-Branch: master Gerrit-Owner: Ori.livneh <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits