This callback is called when urlgrab request fails.
If grab is wrapped in a mirror group, only the mirror
group issues the callback.
---
urlgrabber/grabber.py | 20 ++++++++++++++++++++
urlgrabber/mirror.py | 10 ++++++++++
2 files changed, 30 insertions(+), 0 deletions(-)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 5614bcd..e01a959 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -339,6 +339,15 @@ RETRY RELATED ARGUMENTS
but it cannot (without severe trickiness) prevent the exception
from being raised.
+ failfunc = None
+
+ The callback that gets called when urlgrab request fails.
+ If defined, urlgrab() calls it instead of raising URLGrabError.
+ Callback syntax is identical to failure_callback.
+
+ Contrary to failure_callback, it's called only once. It's primary
+ purpose is to use urlgrab() without a try/except block.
+
interrupt_callback = None
This callback is called if KeyboardInterrupt is received at any
@@ -838,6 +847,7 @@ class URLGrabberOptions:
self.retry = None
self.retrycodes = [-1,2,4,5,6,7]
self.checkfunc = None
+ self.failfunc = None
self.copy_local = 0
self.close_connection = 0
self.range = None
@@ -891,6 +901,12 @@ class URLGrabberOptions:
s = s + indent + '}'
return s
+def _callback(cb, obj):
+ if callable(cb):
+ return cb(obj)
+ cb, arg, karg = cb
+ return cb(obj, *arg, **karg)
+
class URLGrabber(object):
"""Provides easy opening of URLs with a variety of options.
@@ -1023,6 +1039,10 @@ class URLGrabber(object):
fo.close()
return filename
+ if opts.failfunc and not hasattr(opts, 'mirror_group'):
+ try: return self._retry(opts, retryfunc, url, filename)
+ except URLGrabError, opts.exception:
+ return _callback(opts.failfunc, opts)
return self._retry(opts, retryfunc, url, filename)
def urlread(self, url, limit=None, **kwargs):
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 8731aed..5f0120f 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,6 +91,7 @@ import random
import thread # needed for locking to make this threadsafe
from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _callback
def _(st):
return st
@@ -391,6 +392,10 @@ class MirrorGroup:
grabber = mirrorchoice.get('grabber') or self.grabber
func_ref = getattr(grabber, func)
if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
+ # set 'mirror_group' so:
+ # - blocking urlgrab() ignores failfunc
+ # - async urlgrab() can iterate mirrors
+ kwargs['mirror_group'] = self, gr, mirrorchoice
try:
return func_ref( *(fullurl,), **kwargs )
except URLGrabError, e:
@@ -406,6 +411,11 @@ class MirrorGroup:
kw = dict(kwargs)
kw['filename'] = filename
func = 'urlgrab'
+ if kw.get('failfunc'):
+ try: return self._mirror_try(func, url, kw)
+ except URLGrabError, e:
+ opts = CallbackObject(url=url, exception=e, **kw)
+ return _callback(opts.failfunc, opts)
return self._mirror_try(func, url, kw)
def urlopen(self, url, **kwargs):
--
1.7.4.4
_______________________________________________
Yum-devel mailing list
[email protected]
http://lists.baseurl.org/mailman/listinfo/yum-devel
