While libcurl is (supposed to be) thread-safe, pycurl.Curl() object cannot
be shared between multiple threads.
---
urlgrabber/grabber.py | 14 ++++++++------
1 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f6f57bd..26c0d0b 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1458,7 +1458,11 @@ class PyCurlFileObject(object):
raise err
def _do_open(self):
- self.curl_obj = _curl_cache
+ # Curl() object is thread-local
+ i = thread.get_ident()
+ if not i in _curl_cache:
+ _curl_cache[i] = pycurl.Curl()
+ self.curl_obj = _curl_cache[i]
self.curl_obj.reset() # reset all old settings away, just in case
# setup any ranges
self._set_opts()
@@ -1769,14 +1773,12 @@ class PyCurlFileObject(object):
urllib.addinfourl, via. urllib.URLopener.* """
return self.url
-_curl_cache = pycurl.Curl() # make one and reuse it over and over and over
+_curl_cache = {}
def reset_curl_obj():
"""To make sure curl has reread the network/dns info we force a reload"""
- global _curl_cache
- _curl_cache.close()
- _curl_cache = pycurl.Curl()
-
+ for c in _curl_cache.values(): c.close()
+ _curl_cache.clear()
--
1.7.4.4
_______________________________________________
Yum-devel mailing list
[email protected]
http://lists.baseurl.org/mailman/listinfo/yum-devel
