Request for comments - concurrent ssh client

2009-11-04 Thread mk

Hello everyone,

Since I'm not happy with shmux or pssh, I wrote my own concurrent ssh 
program for parallel execution of SSH commands on multiple hosts. Before 
I release program to the wild, I would like to hear (constructive) 
comments on what may be wrong with the program and/or how to fix it. 
(note: the program requires paramiko ssh client module)


#!/usr/local/bin/python -W ignore::DeprecationWarning

import time
import sys
import os
import operator
import paramiko
import threading
import subprocess
import optparse

usage = Usage: cssh [options] IP1 hostname2 IP3 hostname4 
...\n\n(IPs/hostnames on the commandline are actually optional, they can 
be specified in the file, see below.)

op = optparse.OptionParser(usage=usage)

op.add_option('-c','--cmd',dest='cmd',help=Command to run. Mutually 
exclusive with -s.)
op.add_option('-s','--script',dest='script',help=Script file to run. 
Mutually exclusive with -c. Script can have its own arguments, specify 
them in doublequotes, like script -arg arg.)
op.add_option('-i','--script-dir',dest='scriptdir',help=The directory 
where script will be copied and executed. Defaults to /tmp.)
op.add_option('-l','--cleanup',dest='cleanup',action='store_true',help=Delete 
the script on remote hosts after executing it.)
op.add_option('-f','--file',dest='file',help=File with hosts to use, 
one host per line. Concatenated with list of hosts/IP addresses 
specified at the end of the commandline. Optionally, in a line of the 
file you can specify sequence: Address/Hostname Username Password 
SSH_Port separated by spaces (additional parameters can be specified on 
a subset of lines; where not specified, relevant parameters take default 
values).)
op.add_option('-d','--dir',dest='dir',help='Directory for storing 
standard output and standard error of command. If specified, directory 
will be created, with subdirs named IPs/hostnames and relevant files 
stored in those subdirs.')
op.add_option('-u','--username',dest='username',help=Username to 
specify for SSH. Defaults to 'root'.)
op.add_option('-p','--password',dest='password',help=Password. 
Password is used first; if connection fails using password, cssh uses 
SSH key (default or specified).)

op.add_option('-o','--port',dest='port',help=Default SSH port.)
op.add_option('-k','--key',dest='key',help=SSH Key file. Defaults to 
'/root/.ssh/id_dsa'.)
op.add_option('-n','--nokey',dest='nokey',action=store_true, 
help=Turns off using SSH key.)
op.add_option('-t','--timeout',dest='timeout',help=SSH connection 
timeout. Defaults to 20 seconds.)
op.add_option('-m','--monochromatic',dest='mono',action='store_true',help=Do 
not use colors while printing output.)
op.add_option('-r','--maxthreads',dest='maxthreads',help=Maximum 
number of threads working concurrently. Default is 100. Exceeding 200 is 
generally not recommended due to potential exhaustion of address space 
(each thread can use 10 MB of address space and 32-bit systems have a 
maximum of 4GB of address space).)
op.add_option('-q','--quiet',dest='quiet',action='store_true',help=Quiet. 
Do not print out summaries like IPs for which communication succeeded or 
failed, etc.)


# add resource file?

(opts, args) = op.parse_args()

failit = False

if opts.cmd == None and opts.script == None:
print You have to specify one of the following: command to run, 
using -c command or --cmd command, or script to run, using -s scriptfile 
or --script scriptfile.

print
failit = True

if opts.cmd != None and opts.script != None:
print Options command (-c) and script (-s) are mutually exclusive. 
Specify either one.

print
failit = True

if opts.cmd == None and opts.script != None:
try:
scriptpath = opts.script.split()[0]
scriptfo = open(scriptpath,'r')
scriptfo.close()
except IOError:
print Could not open script file %s. % opts.script
print
failit = True

if opts.file == None and args == []:
print You have to specify at least one of the following:
print  - list of IPs/hostnames at the end of the command line 
(after all options)
print  - list of IPs/hostnames stored in file specified after -f 
or --file option (like: -f hostnames.txt)
print  You can also specify both sources. In that case IP/hostname 
lists will be concatenated.

print
failit = True

if opts.password == None and opts.nokey:
print Since using key has been turned off using -n option, you 
have to specify password using -p password or --password password.

print
failit = True

if opts.key is not None and opts.nokey:
print Options -n and -k keyfile are mutually exclusive. Specify 
either one.

print
failit = True

if failit:
sys.exit(0)

if opts.scriptdir == None:
opts.scriptdir = '/tmp'

if opts.cleanup == None:
opts.cleanup = False

if opts.key == None:
opts.key = '/root/.ssh/id_dsa'

if opts.port == None:
opts.port = 22

if opts.nokey:
opts.key = None

if opts.timeout == 

Re: Request for comments - concurrent ssh client

2009-11-04 Thread MRAB

mk wrote:

Hello everyone,

Since I'm not happy with shmux or pssh, I wrote my own concurrent ssh 
program for parallel execution of SSH commands on multiple hosts. Before 
I release program to the wild, I would like to hear (constructive) 
comments on what may be wrong with the program and/or how to fix it. 
(note: the program requires paramiko ssh client module)



[snip]

if opts.cmd == None and opts.script == None:
print You have to specify one of the following: command to run, 
using -c command or --cmd command, or script to run, using -s scriptfile 
or --script scriptfile.

print
failit = True


The normal way to test for None is to use is None or is not None.
You do actually do that in some places!

[snip]

hosts = [ s.strip() for s in hosts if s != '' and s != None and s != '\n' ]
hosts = [ s.split() for s in hosts ]


[snip]
Both '' and None are treated as false by 'if' and 'while'; non-empty
strings are treated as true. Also, s.split() splits on whitespace and
disregards leading and trailing whitespace. The preceding lines can be
simplified to:

hosts = [ s.split() for s in hosts if s and s != '\n' ]


if hosts == []:

[snip]

Empty lists are also treated as false and non-empty lists as true. The
Pythonic way to write the preceding line is:

  if not hosts:

[snip]

scriptcomp = script.strip().split()


As mentioned earlier, .strip().split() can be simplified to just
.split():

  scriptcomp = script.split()

[snip]

try:
self.conobj.connect(self.ip, username=self.username, 
password=self.passw, port=self.port, timeout=opts.timeout, 
allow_agent=False, look_for_keys = False)

loginsuccess = True
except:
pass

[snip]
Avoid bare except wherever possible. In this case you're ignoring
_every_ exception that might occur. VERY BAD IDEA!


def execcmds(self):
so = se = ''
try:
si, so, se = self.conobj.exec_command(self.cmd)
sol = so.readlines()
sel = se.readlines()
so = ''.join([ s.replace('\r\n','\n') for s in sol ])
se = ''.join([ s.replace('\r\n','\n') for s in sel ])

[snip]

Recent versions of Python will accept generator expressions here, so
instead of iterating through a list and creating a new list, which is
then passed to '.join', you can let '.join' do the iterating. This can
save time and memory:

 so = ''.join( s.replace('\r\n','\n') for s in sol )
 se = ''.join( s.replace('\r\n','\n') for s in sel )


if sol != [] or sel != []:


As mentioned earlier, non-empty lists are treated as true:

 if sol or sel:

  def chmodscript(self):
# just in case, as sometimes and on some operating systems the 
execution flags on the script are not always set

self.execcmdonscript('chmod 0755 %s' % self.rspath)


[snip]

You could also use:

 os.chmod(self.rspath, 0755)


def delscript(self):
self.execcmdonscript('rm -f %s' % self.rspath)


[snip]

You could also use:

 os.remove(self.rspath)


for ip, th in queue:
if th.finished:
th.sshclose()
th.join()
thfinished.append((ip,th))
queue.remove((ip,th))


Never modify a list (or any container, in fact) while iterating through
it. Some containers handle iteration by using an index, but if you
remove an item from the container then the index might no longer be
correct. Instead, build a new list of the items you want to keep:

 new_queue = []
 for ip, th in queue:
 if th.finished:
 th.sshclose()
 th.join()
 thfinished.append((ip,th))
 else:
 new_queue.append((ip,th))
 queue = new_queue

If there are other references to the queue itself then it might be
better to replace the contents of the existing queue with those of the
new one instead by changing:

queue = new_queue

to:

queue[:] = new_queue
--
http://mail.python.org/mailman/listinfo/python-list