Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/174592
Change subject: Make the SSH user and authentication socket configurable
......................................................................
Make the SSH user and authentication socket configurable
* Add a config key, 'ssh_user', which defaults to the current username.
* Add a config key, 'ssh_auth_sock', which is unset by default. If set,
it will override SSH_AUTH_SOCK in the environment. This is done before
_assert_auth_sock is called.
* Pass ssh_user to Job and from there to cluster_ssh.
* ssh_auth_sock doesn't need to be passed around. If it is set, it is set in
the environment, which every child process automatically inherits.
Change-Id: I5782e8cbe24bbd28d38967bc16f5f44721dc646f
---
M scap.cfg
M scap/cli.py
M scap/config.py
M scap/main.py
M scap/ssh.py
M scap/tasks.py
6 files changed, 34 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/92/174592/1
diff --git a/scap.cfg b/scap.cfg
index 5a285fe..e3a7a68 100644
--- a/scap.cfg
+++ b/scap.cfg
@@ -72,6 +72,8 @@
statsd_host: statsd.eqiad.wmnet
tcpircbot_host: neon.wikimedia.org
udp2log_host: fluorine.eqiad.wmnet
+ssh_auth_sock: /run/keyholder/proxy.sock
+ssh_user: mwdeploy
[eqiad.wmflabs]
diff --git a/scap/cli.py b/scap/cli.py
index 52bd532..a02b366 100644
--- a/scap/cli.py
+++ b/scap/cli.py
@@ -168,6 +168,12 @@
"""Setup logging."""
log.setup_loggers(self.config, self.arguments.loglevel)
+ def _setup_environ(self):
+ """Setup shell environment."""
+ auth_sock = self.config.get('ssh_auth_sock')
+ if auth_sock is not None:
+ os.environ['SSH_AUTH_SOCK'] = auth_sock
+
def main(self, *extra_args):
"""Main business logic of the application.
@@ -263,6 +269,7 @@
app.arguments = args
app._load_config()
app._setup_loggers()
+ app._setup_environ()
exit_status = app.main(extra_args)
except SystemExit as ex:
diff --git a/scap/config.py b/scap/config.py
index f5e5028..b3c8766 100644
--- a/scap/config.py
+++ b/scap/config.py
@@ -6,6 +6,7 @@
"""
import ConfigParser
+import getpass
import os
import socket
@@ -23,6 +24,7 @@
'udp2log_host': None,
'udp2log_port': '8420',
'wmf_realm': 'production',
+ 'ssh_user': getpass.getuser(),
'datacenter': 'pmtpa',
}
diff --git a/scap/main.py b/scap/main.py
index 72c589c..63fc0fa 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -40,7 +40,7 @@
# Update proxies
proxies = self._get_proxy_list()
with log.Timer('sync-proxies', self.get_stats()):
- update_proxies = ssh.Job(proxies)
+ update_proxies = ssh.Job(proxies, user=self.config['ssh_user'])
update_proxies.command(self._proxy_sync_command())
update_proxies.progress('sync-proxies')
succeeded, failed = update_proxies.run()
@@ -51,7 +51,8 @@
# Update apaches
with log.Timer('sync-apaches', self.get_stats()):
- update_apaches = ssh.Job(self._get_apache_list())
+ update_apaches = ssh.Job(self._get_apache_list(),
+ user=self.config['ssh_user'])
update_apaches.exclude_hosts(proxies)
update_apaches.shuffle()
update_apaches.command(self._apache_sync_command(proxies))
@@ -223,7 +224,8 @@
def _after_cluster_sync(self):
# Ask apaches to rebuild l10n CDB files
with log.Timer('scap-rebuild-cdbs', self.get_stats()):
- rebuild_cdbs = ssh.Job(self._get_apache_list())
+ rebuild_cdbs = ssh.Job(self._get_apache_list(),
+ user=self.config['ssh_user'])
rebuild_cdbs.shuffle()
rebuild_cdbs.command('sudo -u mwdeploy -n -- %s' %
self.get_script_path('scap-rebuild-cdbs'))
diff --git a/scap/ssh.py b/scap/ssh.py
index 11788c1..a8389a2 100644
--- a/scap/ssh.py
+++ b/scap/ssh.py
@@ -24,10 +24,11 @@
"""Execute a job on a group of remote hosts via ssh."""
_logger = None
- def __init__(self, hosts=None, command=None):
+ def __init__(self, hosts=None, command=None, user=None):
self.hosts(hosts or [])
self._command = command
self._reporter = None
+ self._user = user
def get_logger(self):
"""Lazy getter for a logger instance."""
@@ -88,7 +89,8 @@
if self._reporter:
return self._run_with_reporter(batch_size)
else:
- return list(cluster_ssh(self._hosts, self._command, batch_size))
+ return list(cluster_ssh(self._hosts, self._command, self._user,
+ batch_size))
def _run_with_reporter(self, batch_size):
"""Run job and feed results to a :class:`log.ProgressReporter` as they
@@ -97,7 +99,7 @@
self._reporter.start()
for host, status, output in cluster_ssh(
- self._hosts, self._command, batch_size):
+ self._hosts, self._command, self._user, batch_size):
if status == 0:
self._reporter.add_success()
else:
@@ -108,7 +110,7 @@
return self._reporter.ok, self._reporter.failed
-def cluster_ssh(hosts, command, limit=80):
+def cluster_ssh(hosts, command, user=None, limit=80):
"""Run a command via SSH on multiple hosts concurrently."""
hosts = set(hosts)
@@ -124,7 +126,11 @@
while hosts or procs:
if hosts and len(procs) < limit:
host = hosts.pop()
- ssh_command = SSH + (host,) + tuple(command)
+ ssh_command = list(SSH)
+ if user:
+ ssh_command.append('-l%s' % user)
+ ssh_command.append(host)
+ ssh_command.extend(command)
proc = subprocess.Popen(ssh_command, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, preexec_fn=os.setsid)
procs[proc.pid] = (proc, host)
diff --git a/scap/tasks.py b/scap/tasks.py
index 71769cc..56c1187 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -226,8 +226,13 @@
# Purge from deploy directroy across cluster
# --force option given to rm to ignore missing files as before
+<<<<<<< HEAD
purge = ssh.Job().role('mediawiki-installation')
purge.command('sudo -u mwdeploy -n -- /bin/rm '
+=======
+ purge = ssh.Job(user=cfg['ssh_user']).role('mediawiki-installation')
+ purge.command('sudo -u mwdeploy /bin/rm '
+>>>>>>> Make the SSH user and authentication socket configurable
'--recursive --force %s/*' % deployed_l10n)
purge.progress('l10n purge').run()
@@ -301,8 +306,8 @@
with log.Timer('sync_wikiversions', stats):
compile_wikiversions_cdb('stage', cfg)
- rsync = ssh.Job(hosts).shuffle()
- rsync.command('sudo -u mwdeploy -n -- /usr/bin/rsync -l '
+ rsync = ssh.Job(hosts, user=cfg['ssh_username']).shuffle()
+ rsync.command('sudo -u mwdeploy /usr/bin/rsync -l '
'%(master_rsync)s::common/wikiversions*.{json,cdb} '
'%(deploy_dir)s' % cfg)
return rsync.progress('sync_wikiversions').run()
--
To view, visit https://gerrit.wikimedia.org/r/174592
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5782e8cbe24bbd28d38967bc16f5f44721dc646f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Ori.livneh <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
