I want to get the contents of a list of URLs, and if the getPage is
successful then I want to write it to a file. If it fails, I want to log a
message. In addition, I would like to have this process run every five
minutes.
The problem is how to specify a yes/no scenario when the callbacks take
arguments. Also, is the best way to loop to just put FeederProtocol.start()
in a loop? any ehlp would be greatly appreciated.
while true:
get page
if error:
log
else:
write to file
sleep 5 minutes
class FeederProtocol(object):
def get_page_error(self, _failure, log, feed_name):
""" Failure while getting page. """
msg = 'While getting %s: %s' % (feed_name, _failure.getTraceback())
log.error(msg)
_failure.trap(RuntimeError) # Caused chain to stop
raise(RuntimeError)
get_page(url):
return client.getPage(url, timeout=30)
def page_to_file(self, feed, name):
try:
fnout = os.path.join(app_conf['download_dir'], name)
fout = open(fnout, 'w')
except IOError, e:
log.error('Opening %s: %s' % (fnout, e))
return
try:
fout.write(feed)
except IOError, e:
log.error('Wrting to %s: %s' % (fnout, e))
finally:
fout.close()
def start(self, urls):
for url in urls:
d = defer.succeed(self.printStatus())
d.addCallback(self.get_page, feed, log, app_conf)
d.addCallbacks((self.page_to_file, log, app_conf), (self.get_page_error,
log, feed[0]))
class FeederFactory(protocol.ClientFactory):
protocol = FeederProtocol()
def __init__(self, url_list):
urls = url_list
self.start(self.urls)
def start(self, urls):
protocol.start(urls)
if __name__=="__main__":
name = 'mytest'
urls = ['www.example.com', 'www.example2.com']
f = FeederFactory(urls)
reactor.run()
_______________________________________________
Twisted-web mailing list
[email protected]
http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-web
