Make the removal of a failed mirror in async path conditional,
so it's in line with synchronous urlgrab. Mirror failures are
counted and the mirror selection always uses mirror with least
failures. (overrides speed estimation)
---
urlgrabber/grabber.py | 21 +++++++++++++++------
urlgrabber/mirror.py | 6 +++++-
2 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 057cbbe..83823ea 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2176,13 +2176,18 @@ def parallel_wait(meter = 'text'):
continue
if opts.mirror_group:
- mg, failed = opts.mirror_group
+ mg, failed, removed = opts.mirror_group
+ failed[key] = failed.get(key, 0) + 1
opts.mirror = key
opts.exception = ug_err
- action = _run_callback(mg.failure_callback, opts)
- if not (action and action.get('fail')):
+ action = mg.default_action or {}
+ if mg.failure_callback:
+ opts.tries = sum(failed.values())
+ action.update(_run_callback(mg.failure_callback, opts))
+ if not action.get('fail', 0):
# mask this mirror and retry
- failed.add(key)
+ if action.get('remove', 1):
+ removed.add(key)
_async_queue.append(opts)
continue
@@ -2209,17 +2214,21 @@ def parallel_wait(meter = 'text'):
perform()
if opts.mirror_group:
- mg, failed = opts.mirror_group
+ mg, failed, removed = opts.mirror_group
# find the best mirror
best = None
+ best_speed = None
for mirror in mg.mirrors:
key = mirror['mirror']
- if key in failed: continue
+ if key in removed: continue
# estimate mirror speed
speed = _TH.estimate(key)
speed /= 1 + host_con.get(key, 0)
+
+ # 2-tuple to select mirror with least failures
+ speed = -failed.get(key, 0), speed
if best is None or speed > best_speed:
best = mirror
best_speed = speed
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 675c2bf..ac78b34 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -188,6 +188,7 @@ class MirrorGroup:
obj.exception = < exception that was raised >
obj.mirror = < the mirror that was tried >
+ obj.tries = < the number of mirror tries so far >
obj.relative_url = < url relative to the mirror >
obj.url = < full url that failed >
# .url is just the combination of .mirror
@@ -387,7 +388,9 @@ class MirrorGroup:
try: del kw[k]
except KeyError: pass
+ tries = 0
while 1:
+ tries += 1
mirrorchoice = self._get_mirror(gr)
fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
kwargs = dict(mirrorchoice.get('kwargs', {}))
@@ -404,6 +407,7 @@ class MirrorGroup:
obj.mirror = mirrorchoice['mirror']
obj.relative_url = gr.url
obj.url = fullurl
+ obj.tries = tries
self._failure(gr, obj)
def urlgrab(self, url, filename=None, **kwargs):
@@ -411,7 +415,7 @@ class MirrorGroup:
kw['filename'] = filename
if kw.get('async'):
# enable mirror failovers in async path
- kw['mirror_group'] = self, set()
+ kw['mirror_group'] = self, {}, set()
kw['relative_url'] = url
else:
kw.pop('failfunc', None)
--
1.7.4.4
_______________________________________________
Yum-devel mailing list
[email protected]
http://lists.baseurl.org/mailman/listinfo/yum-devel
