Ran into a weird issue using python on cygwin. When trying to use the stdout handle from a subprocess spawned using the subprocess.Popen class, it hangs when accessed via a thread. This behavior does /not/ occur on other platforms -- nor when using os.popen from threads on cygwin. Nor when using subprocess.Popen from cygwin without threads.
I know I could avoid this by using subprocess.Popen(...).communicate(), but in my /real/ application, I want line-by-line access to the sub's output, whereas communicate() gobbles the entire output and returns a list of strings for stdout, and for stderr. From the help text: communicate() returns a tuple (stdout, stderr). Note: The data read is buffered in memory, so do not use this method if the data size is large or unlimited. So, I don't want to do /that/. I've attached a test case to demonstrate the issue. I was hoping (a) somebody has seen this before, and has a simple solution for me, or (b) the python maintainer (Jason?) could take a look, and maybe raise a bug report upstream. I'm using 2.5.1-2, but I see the same behavior on cygwin using 2.5.2. When run on native windows, this test case requires Fping.exe (http://www.kwakkelflap.com/fping.html) because -- at least on Vista -- the native ping is b0rked. When native ping is called on Vista from a threaded context, if a host is unreachable, it reports summary info for the previously-accessed reachable host. Very strange -- but not related to the problem at hand. For cygwin, of course, the test case uses /usr/bin/ping. Test case --help output: Usage: ./testprog.py [htsS] Tests os.popen vs. subprocess.Popen for various platforms On native windows: requires Fping.exe (http://www.kwakkelflap.com/fping.html) In operation, this program 'pings' ten IP addresses in the current network, including localhost. It is used to demonstrate that the .stdout member of subprocess.Popen() does not work on Cygwin when using threads -h, --help print this message -t, --threads use threads (default is seqential) -s, --subprocess use subprocess.Popen().stdout (default is os.popen) -S, --shell when using subprocess.Popen(), execute the target using the shell (/bin/sh or cmd.exe), just like os.popen does. default for subprocess.Popen is to use an execv-style list. Behavior: native windows cygwin unix <no args> (os.popen, sequential) OK OK OK -t (os.popen, threaded) OK OK OK -s -S (subprocess, sequential, via shell) OK OK OK -s (subprocess, sequential, execv) OK OK OK -t -s -S (subprocess, threaded, via shell) OK hangs OK -t -s (subprocess, threaded, execv) OK hangs OK
#!/usr/bin/python """ Usage: %s [htsS] Tests os.popen vs. subprocess.Popen for various platforms On native windows: requires Fping.exe (http://www.kwakkelflap.com/fping.html) In operation, this program 'pings' ten IP addresses in the current network, including localhost. It is used to demonstrate that the .stdout member of subprocess.Popen() does not work on Cygwin when using threads -h, --help print this message -t, --threads use threads (default is seqential) -s, --subprocess use subprocess.Popen().stdout (default is os.popen) -S, --shell when using subprocess.Popen(), execute the target using the shell (/bin/sh or cmd.exe), just like os.popen does. default for subprocess.Popen is to use an execv-style list. Behavior: native windows cygwin unix <no args> (os.popen, sequential) OK OK OK -t (os.popen, threaded) OK OK OK -s -S (subprocess, sequential, via shell) OK OK OK -s (subprocess, sequential, execv) OK OK OK -t -s -S (subprocess, threaded, via shell) OK hangs OK -t -s (subprocess, threaded, execv) OK hangs OK """ import os import re import time import sys import getopt from threading import Thread import socket import struct import subprocess # NOTE: OPT_use_shell is always treated as True if OPT_use_subprocess is False # (that is, os.popen always uses the shell!) OPT_use_threads=False OPT_use_subprocess=False OPT_use_shell=False if sys.platform == 'win32': def ping_cmdline(ip, shell=False): # windows ping is b0rked on Vista, so we're forced to use a different # ping program. Can't use cygwin ping, must be a native program. # Fping from http://www.kwakkelflap.com/fping.html seems pretty # good, is virus free, and is not b0rked. if shell: return "Fping.exe " + ip + " -i -n 2" return ["Fping.exe ", ip, "-i", "-n", "2"] elif sys.platform == 'cygwin': def ping_cmdline(ip, shell=False): # cygwin's ping is also odd if shell: return "ping.exe -q " + ip + " 56 2" return ["ping.exe", "-q", ip, "56", "2"] else: def ping_cmdline(ip, shell=False): if shell: return "ping -q -c2 " + ip return ["ping", "-q", "-c2", ip] def get_pipe_subprocess_Popen(ip): global OPT_use_shell print >>sys.stderr, "Using subprocess.Popen (shell=%s)" % OPT_use_shell chld_stdin = os.open(os.devnull, os.O_RDONLY); return subprocess.Popen(ping_cmdline(ip, shell=OPT_use_shell), shell=OPT_use_shell, stdin=chld_stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout def get_pipe_os_popen(ip): global OPT_use_shell print >>sys.stderr, "Using os.popen (shell=%s)" % OPT_use_shell return os.popen(ping_cmdline(ip, shell=OPT_use_shell), "r") def get_pipe(ip): global OPT_use_subprocess if OPT_use_subprocess: return get_pipe_subprocess_Popen(ip) else: return get_pipe_os_popen(ip) class testit(Thread): def __init__ (self,ip): Thread.__init__(self) self.ip = ip self.status = -1 def run(self): pingaling = get_pipe(self.ip) while 1: line = pingaling.readline() if not line: break igot = re.findall(testit.lifeline,line) if igot: self.status = int(igot[0]) if sys.platform == 'win32': testit.lifeline = re.compile(r"Received = (\d)") lifeline = re.compile(r"Received = (\d)") elif sys.platform == 'cygwin': testit.lifeline = re.compile(r"(\d) packets received") lifeline = re.compile(r"(\d) packets received") else: testit.lifeline = re.compile(r"(\d) received") lifeline = re.compile(r"(\d) received") report = ("No response","Partial Response","Alive","Internal Error") def naive_get_ip_addr(): return socket.gethostbyname(socket.gethostname()) def dottedQuadToNum(ip): "convert decimal dotted quad string to long integer" return struct.unpack('!L',socket.inet_aton(ip))[0] def numToDottedQuad(n): "convert long int to dotted quad string" return socket.inet_ntoa(struct.pack('!L',n)) def makeMask(n): "return a mask of n bits as a long integer" return (1L<<n)-1 def ipToNetAndHost(ip, maskbits): "returns tuple (network, host) dotted-quad addresses given IP and mask size" # (by Greg Jorgensen) n = dottedQuadToNum(ip) m = makeMask(maskbits) host = n & m net = n - host return numToDottedQuad(net), numToDottedQuad(host) def main_threads(hosts): print >>sys.stderr, "Using threads" pinglist = [] for ip in hosts: current = testit(ip) pinglist.append(current) current.start() for pingle in pinglist: pingle.join() print "Status from ",pingle.ip,"is",report[pingle.status] def main_nothreads(hosts): print >>sys.stderr, "No threads" for ip in hosts: pingaling = get_pipe(ip) while 1: line = pingaling.readline() if not line: break igot = re.findall(lifeline,line) if igot: print "Status from ",ip," is ",report[int(igot[0])] def main(argv=None): global OPT_use_threads, OPT_use_subprocess, OPT_use_shell if argv is None: argv = sys.argv opts, args = getopt.getopt(argv[1:], "htsS",["help", "threads", "subprocess", "shell"]) for opt, value in opts: if opt in ('-h', '--help'): print __doc__ % argv[0] return 0 if opt in ('-t', '--threads'): OPT_use_threads = True if opt in ('-s', '--subprocess'): OPT_use_subprocess = True if opt in ('-S', '--shell'): OPT_use_shell = True # NOTE: if using os.popen (that is, OPT_use_subprocess is false) # then we *always* use the shell if not OPT_use_subprocess: OPT_use_shell = True # compute range of ip addresses myip = naive_get_ip_addr() (netwk, hst) = ipToNetAndHost(myip, 8) hstd = dottedQuadToNum(hst) netwkd = dottedQuadToNum(netwk) if hstd >= 250: hostrange = range(245,254) else: hostrange = range(hstd - (hstd % 10) + 1, hstd - (hstd % 10) + 11) hosts = [] for host in hostrange: ip = numToDottedQuad(netwkd + host) hosts.append(ip) print time.ctime() if OPT_use_threads: main_threads(hosts) else: main_nothreads(hosts) print time.ctime() if __name__ == "__main__": sys.exit(main())
-- Unsubscribe info: http://cygwin.com/ml/#unsubscribe-simple Problem reports: http://cygwin.com/problems.html Documentation: http://cygwin.com/docs.html FAQ: http://cygwin.com/faq/