Still freezing sometimes, like 1 out of 10 times that I run it. Here is updated code and a couple of outputs.
################ code #!/usr/bin/env python import sys import Queue import multiprocessing import time def FOO(a, b, c): print 'foo', a, b, c return (a + b) * c class MyWorker(multiprocessing.Process): def __init__(self, name, inbox, outbox): super(MyWorker, self).__init__() self.name = name self.inbox = inbox self.outbox = outbox print >> sys.stderr, 'Created %s' % self.name; sys.stderr.flush() def run(self): print >> sys.stderr, 'Running %s' % self.name; sys.stderr.flush() while True: try: args = self.inbox.get_nowait() print >> sys.stderr, '%s got something to do' % self.name; sys.stderr.flush() except Queue.Empty: break self.outbox.put(FOO(*args)) if __name__ == '__main__': # This file is being run as the main script. This part won't be # run if the file is imported. print >> sys.stderr, 'Creating todo queue'; sys.stderr.flush() todo = multiprocessing.Queue() for i in xrange(100): todo.put((i, i+1, i+2)) print >> sys.stderr, 'Creating results queue'; sys.stderr.flush() result_queue = multiprocessing.Queue() print >> sys.stderr, 'Creating Workers'; sys.stderr.flush() w1 = MyWorker('Worker 1', todo, result_queue) w2 = MyWorker('Worker 2', todo, result_queue) print >> sys.stderr, 'Starting Worker 1'; sys.stderr.flush() w1.start() print >> sys.stderr, 'Starting Worker 2'; sys.stderr.flush() w2.start() for i in xrange(100): print result_queue.get() ################ output 1 (I ctrl-c'd it after it froze) Creating todo queue Creating results queue Creating Workers Created Worker 1 Created Worker 2 Starting Worker 1 Running Worker 1 Starting Worker 2 Running Worker 2 Traceback (most recent call last): File "./multi.py", line 53, in <module> print result_queue.get() File "/home/frede00e/software/python/lib/python2.7/multiprocessing/queues.py", line 91, in get res = self._recv() KeyboardInterrupt ################### output 2 Creating todo queue Creating results queue Creating Workers Created Worker 1 Created Worker 2 Starting Worker 1 Running Worker 1 Worker 1 got something to do Starting Worker 2 foo 0 1 2 Worker 1 got something to do foo 1 2 3 Worker 1 got something to do foo 2 3 4 Worker 1 got something to do foo 3 4 5 Worker 1 got something to do foo 4 5 6 Worker 1 got something to do foo 5 6 7 Worker 1 got something to do foo 6 7 8 Worker 1 got something to do foo 7 8 9 Worker 1 got something to do foo 8 9 10 Worker 1 got something to do foo 9 10 11 Worker 1 got something to do foo 10 11 12 Worker 1 got something to do foo 11 12 13 Worker 1 got something to do foo 12 13 14 Worker 1 got something to do foo 13 14 15 Worker 1 got something to do foo 14 15 16 Worker 1 got something to do foo 15 16 17 Worker 1 got something to do foo 16 17 18 Worker 1 got something to do foo 17 18 19 Running Worker 2 2 9 20 35 54 77 104 135 170 209 252 299 350 405 464 527 594 665 Traceback (most recent call last): File "./multi.py", line 53, in <module> print result_queue.get() File "/home/frede00e/software/python/lib/python2.7/multiprocessing/queues.py", line 91, in get res = self._recv() KeyboardInterrupt On Fri, Feb 24, 2012 at 1:36 PM, MRAB <pyt...@mrabarnett.plus.com> wrote: > On 24/02/2012 17:00, Eric Frederich wrote: > >> I can sill get it to freeze and nothing is printed out from the other >> except block. >> Does it look like I'm doing anything wrong here? >> >> [snip] > I don't normally use multiprocessing, so I forgot about a critical > detail. :-( > > When the multiprocessing module starts a process, that process > _imports_ the module which contains the function which is to be run, so > what's happening is that when your script is run, it creates and starts > workers, the multiprocessing module makes a new process for each > worker, each of those processes then imports the script, which creates > and starts workers, etc, leading to an ever-increasing number of > processes. > > The solution is to ensure that the script/module distinguishes between > being run as the main script and being imported as a module: > > > #!/usr/bin/env python > > import sys > import Queue > import multiprocessing > import time > > def FOO(a, b, c): > print 'foo', a, b, c > return (a + b) * c > > class MyWorker(multiprocessing.**Process): > def __init__(self, inbox, outbox): > super(MyWorker, self).__init__() > self.inbox = inbox > self.outbox = outbox > print >> sys.stderr, '1' * 80; sys.stderr.flush() > def run(self): > print >> sys.stderr, '2' * 80; sys.stderr.flush() > while True: > try: > args = self.inbox.get_nowait() > except Queue.Empty: > break > self.outbox.put(FOO(*args)) > > if __name__ == '__main__': > # This file is being run as the main script. This part won't be > # run if the file is imported. > > todo = multiprocessing.Queue() > > for i in xrange(100): > todo.put((i, i+1, i+2)) > > print >> sys.stderr, 'a' * 80; sys.stderr.flush() > result_queue = multiprocessing.Queue() > > print >> sys.stderr, 'b' * 80; sys.stderr.flush() > w1 = MyWorker(todo, result_queue) > print >> sys.stderr, 'c' * 80; sys.stderr.flush() > w2 = MyWorker(todo, result_queue) > > print >> sys.stderr, 'd' * 80; sys.stderr.flush() > w1.start() > print >> sys.stderr, 'e' * 80; sys.stderr.flush() > w2.start() > print >> sys.stderr, 'f' * 80; sys.stderr.flush() > > for i in xrange(100): > print result_queue.get() > -- > http://mail.python.org/**mailman/listinfo/python-list<http://mail.python.org/mailman/listinfo/python-list> >
-- http://mail.python.org/mailman/listinfo/python-list