proxy style i tried:
for auth
# 195.39.253.61:8080@SuperVIP79755:2GVkM4MIii
# SuperVIP79755:[email protected]:8080
#
# working for urllib2 module in pyhton
# working for selinium module in python
# but not for scrapy module in pyhton
for binding
# 195.39.253.61:8085
# not working at all
method i am tying
for urllib2
# proxy = urllib2.ProxyHandler({'http': 'http://'+use:pass@ip:port})
auth = urllib2.HTTPBasicAuthHandler()
opener = urllib2.build_opener(proxy, auth, urllib2.HTTPHandler)
urllib2.install_opener(opener)
conn = urllib2.urlopen(url)
for selinum driver
# service_args = [
'--proxy=xxx.xxx.xx.xxx:xxxx',
'--proxy-auth=username:password',
'--proxy-type=http',
]
driver = webdriver.PhantomJS(service_args=service_args)
driver.get(url)
for scrapy
# class ProxyMiddleware(object):
def process_request(self, request, spider):
subprocess.call(['export
http_proxy=http://user:pass@ips:port], shell=True)
request.meta['proxy'] =
"http://195.39.253.61:8080";
proxy_user_pass = user:pass
encoded_user_pass =
base64.encodestring(proxy_user_pass)
request.headers['Proxy-Authorization'] = 'Basic '
+ encoded_user_pass
# seting.py
DOWNLOADER_MIDDLEWARES = {
'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 110,
'my_project_name.proxymiddle.ProxyMiddleware':
100,
}
--
You received this message because you are subscribed to the Google Groups
"scrapy-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/scrapy-users.
For more options, visit https://groups.google.com/groups/opt_out.
