Consolidate LibcloudHTTPConnection and LibcloudHTTPSConnection into a single class. Create a session based wrapper for requests. Implement the SSL verification overridding.
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/30d27bbc Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/30d27bbc Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/30d27bbc Branch: refs/heads/requests Commit: 30d27bbc198098daedaf6d57742ad2e0bad5118e Parents: 10cea10 Author: anthony-shaw <[email protected]> Authored: Tue Mar 29 15:38:23 2016 +1100 Committer: anthony-shaw <[email protected]> Committed: Tue Mar 29 15:38:23 2016 +1100 ---------------------------------------------------------------------- libcloud/common/base.py | 11 ++- libcloud/httplib_ssl.py | 140 +++++++++++-------------------- libcloud/test/test_connection.py | 29 +------ libcloud/test/test_httplib_ssl.py | 4 +- libcloud/utils/loggingconnection.py | 4 +- requirements-tests.txt | 1 + setup.py | 3 +- 7 files changed, 65 insertions(+), 127 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index ab642f3..22852b4 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -45,8 +45,7 @@ from libcloud.utils.compression import decompress_data from libcloud.common.exceptions import exception_from_message from libcloud.common.types import LibcloudError, MalformedResponseError -from libcloud.httplib_ssl import LibcloudHTTPConnection -from libcloud.httplib_ssl import LibcloudHTTPSConnection +from libcloud.httplib_ssl import LibcloudConnection __all__ = [ 'RETRY_FAILED_HTTP_REQUESTS', @@ -319,8 +318,8 @@ class Connection(object): """ A Base Connection class to derive from. """ - # conn_classes = (LoggingHTTPSConnection) - conn_classes = (LibcloudHTTPConnection, LibcloudHTTPSConnection) + # conn_class = LoggingHTTPSConnection + conn_class = LibcloudConnection responseCls = Response rawResponseCls = RawResponse @@ -470,12 +469,12 @@ class Connection(object): if self.proxy_url: kwargs.update({'proxy_url': self.proxy_url}) - connection = self.conn_classes[secure](**kwargs) + connection = self.conn_class(**kwargs) # You can uncoment this line, if you setup a reverse proxy server # which proxies to your endpoint, and lets you easily capture # connections in cleartext when you setup the proxy to do SSL # for you - # connection = self.conn_classes[False]("127.0.0.1", 8080) + # connection = self.conn_class("127.0.0.1", 8080) self.connection = connection http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/libcloud/httplib_ssl.py ---------------------------------------------------------------------- diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py index ce384d5..aea5eb6 100644 --- a/libcloud/httplib_ssl.py +++ b/libcloud/httplib_ssl.py @@ -18,18 +18,15 @@ verification, depending on libcloud.security settings. """ import os import socket -import warnings import requests -import libcloud.security from libcloud.utils.py3 import urlparse __all__ = [ 'LibcloudBaseConnection', - 'LibcloudHTTPConnection', - 'LibcloudHTTPSConnection' + 'LibcloudConnection' ] HTTP_PROXY_ENV_VARIABLE_NAME = 'http_proxy' @@ -141,103 +138,64 @@ class LibcloudBaseConnection(object): proxy_password) -class LibcloudHTTPConnection(LibcloudBaseConnection): - def __init__(self, *args, **kwargs): - # Support for HTTP proxy - proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None) - proxy_url = kwargs.pop('proxy_url', proxy_url_env) - - super(LibcloudHTTPConnection, self).__init__() - - if proxy_url: - self.set_http_proxy(proxy_url=proxy_url) - - def connect(): - pass - - def request(self, method, url, body=None, headers=None): - method = method.lower() - if method == 'get': - response = self.session.get(url, headers=headers) - elif method == 'post': - response = self.session.post(url, data=body, headers=headers) - elif method == 'head': - response = self.session.head(url, headers=headers) - return response - - -class LibcloudHTTPSConnection(LibcloudBaseConnection): - """ - LibcloudHTTPSConnection - - Subclass of HTTPSConnection which verifies certificate names - if and only if CA certificates are available. - """ - verify = True # verify by default - ca_cert = None # no default CA Certificate +class LibcloudConnection(LibcloudBaseConnection): + timeout = None + host = None + response = None - def __init__(self, *args, **kwargs): - """ - Constructor - """ - self._setup_verify() + def __init__(self, host, port, **kwargs): + self.host = '{}://{}'.format( + 'https' if port == 443 else 'http', + host + ) # Support for HTTP proxy proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None) proxy_url = kwargs.pop('proxy_url', proxy_url_env) - super(LibcloudHTTPSConnection, self).__init__() + super(LibcloudConnection, self).__init__() if proxy_url: self.set_http_proxy(proxy_url=proxy_url) - - def _setup_verify(self): - """ - Setup Verify SSL or not - - Reads security module's VERIFY_SSL_CERT and toggles whether - the class overrides the connect() class method or runs the - inherited httplib.HTTPSConnection connect() - """ - self.verify = libcloud.security.VERIFY_SSL_CERT - - if self.verify: - self._setup_ca_cert() - else: - warnings.warn(libcloud.security.VERIFY_SSL_DISABLED_MSG) - - def _setup_ca_cert(self): - """ - Setup CA Certs - - Search in CA_CERTS_PATH for valid candidates and - return first match. Otherwise, complain about certs - not being available. - """ - if not self.verify: - return - - ca_certs_available = [cert - for cert in libcloud.security.CA_CERTS_PATH - if os.path.exists(cert) and os.path.isfile(cert)] - if ca_certs_available: - # use first available certificate - self.ca_cert = ca_certs_available[0] - else: - raise RuntimeError( - libcloud.security.CA_CERTS_UNAVAILABLE_ERROR_MSG) - - def connect(self): + self.session.timeout = kwargs.get('timeout', 60) + + def request(self, method, url, body=None, headers=None, raw=False): + self.response = self.session.request( + method=method.lower(), + url=''.join([self.host, url]), + data=body, + headers=headers, + allow_redirects=1, + stream=raw + ) + + def getresponse(self): + return self + + def getheaders(self): + # urlib decoded response body, libcloud has a bug + # and will not check if content is gzipped, so let's + # remove headers indicating compressed content. + if 'content-encoding' in self.response.headers: + del self.response.headers['content-encoding'] + return self.response.headers + + @property + def status(self): + return self.response.status_code + + @property + def reason(self): + return None if self.response.status_code > 400 else self.response.text + + def connect(self): # pragma: no cover pass - def request(self, method, url, body=None, headers=None): - method = method.lower() - if method == 'get': - response = requests.get(url, headers=headers) - elif method == 'post': - response = requests.post(url, data=body, headers=headers) - elif method == 'head': - response = requests.head(url, headers=headers) - return response + def read(self): + return self.response.content + + def close(self): # pragma: no cover + # return connection back to pool + self.response.close() def get_socket_error_exception(ssl_version, exc): http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/libcloud/test/test_connection.py ---------------------------------------------------------------------- diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py index a588ca6..5a97b45 100644 --- a/libcloud/test/test_connection.py +++ b/libcloud/test/test_connection.py @@ -23,9 +23,8 @@ from mock import Mock, call, patch from libcloud.test import unittest from libcloud.common.base import Connection -from libcloud.utils.loggingconnection import LoggingConnection from libcloud.httplib_ssl import LibcloudBaseConnection -from libcloud.httplib_ssl import LibcloudHTTPConnection +from libcloud.httplib_ssl import LibcloudConnection from libcloud.utils.misc import retry @@ -74,7 +73,7 @@ class BaseConnectionClassTestCase(unittest.TestCase): proxy_url=proxy_url) def test_constructor(self): - conn = LibcloudHTTPConnection(host='localhost', port=80) + conn = LibcloudConnection(host='localhost', port=80) self.assertEqual(conn.proxy_scheme, None) self.assertEqual(conn.proxy_host, None) self.assertEqual(conn.proxy_port, None) @@ -86,7 +85,7 @@ class BaseConnectionClassTestCase(unittest.TestCase): self.assertEqual(conn.proxy_port, 3128) proxy_url = 'http://127.0.0.4:3128' - conn = LibcloudHTTPConnection(host='localhost', port=80, + conn = LibcloudConnection(host='localhost', port=80, proxy_url=proxy_url) self.assertEqual(conn.proxy_scheme, 'http') self.assertEqual(conn.proxy_host, '127.0.0.4') @@ -94,7 +93,7 @@ class BaseConnectionClassTestCase(unittest.TestCase): os.environ['http_proxy'] = proxy_url proxy_url = 'http://127.0.0.5:3128' - conn = LibcloudHTTPConnection(host='localhost', port=80, + conn = LibcloudConnection(host='localhost', port=80, proxy_url=proxy_url) self.assertEqual(conn.proxy_scheme, 'http') self.assertEqual(conn.proxy_host, '127.0.0.5') @@ -259,26 +258,6 @@ class ConnectionClassTestCase(unittest.TestCase): self.assertEqual(con.context, {}) - def test_log_curl(self): - url = '/test/path' - body = None - headers = {} - - con = LoggingConnection() - con.protocol = 'http' - con.host = 'example.com' - con.port = 80 - - for method in ['GET', 'POST', 'PUT', 'DELETE']: - cmd = con._log_curl(method=method, url=url, body=body, - headers=headers) - self.assertEqual(cmd, 'curl -i -X %s --compress http://example.com:80/test/path' % - (method)) - - # Should use --head for head requests - cmd = con._log_curl(method='HEAD', url=url, body=body, headers=headers) - self.assertEqual(cmd, 'curl -i --head --compress http://example.com:80/test/path') - def _raise_socket_error(self): raise socket.gaierror('') http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/libcloud/test/test_httplib_ssl.py ---------------------------------------------------------------------- diff --git a/libcloud/test/test_httplib_ssl.py b/libcloud/test/test_httplib_ssl.py index 996498f..8f5885d 100644 --- a/libcloud/test/test_httplib_ssl.py +++ b/libcloud/test/test_httplib_ssl.py @@ -24,7 +24,7 @@ from mock import patch import libcloud.security from libcloud.utils.py3 import reload -from libcloud.httplib_ssl import LibcloudHTTPSConnection +from libcloud.httplib_ssl import LibcloudConnection from libcloud.test import unittest @@ -36,7 +36,7 @@ class TestHttpLibSSLTests(unittest.TestCase): def setUp(self): libcloud.security.VERIFY_SSL_CERT = False libcloud.security.CA_CERTS_PATH = ORIGINAL_CA_CERS_PATH - self.httplib_object = LibcloudHTTPSConnection('foo.bar') + self.httplib_object = LibcloudConnection('foo.bar') def test_custom_ca_path_using_env_var_doesnt_exist(self): os.environ['SSL_CERT_FILE'] = '/foo/doesnt/exist' http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/libcloud/utils/loggingconnection.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/loggingconnection.py b/libcloud/utils/loggingconnection.py index 2b6550d..f6926e8 100644 --- a/libcloud/utils/loggingconnection.py +++ b/libcloud/utils/loggingconnection.py @@ -19,7 +19,7 @@ except: import json from pipes import quote as pquote -import xml.dom.minidom +from xml.dom.minidom import parseString import sys import os @@ -104,7 +104,7 @@ class LoggingConnection(): pass elif pretty_print and content_type == 'text/xml': try: - elem = xml.dom.minidom.parseString(body.decode('utf-8')) + elem = parseString(body.decode('utf-8')) body = elem.toprettyxml() except Exception: # Invalid XML http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/requirements-tests.txt ---------------------------------------------------------------------- diff --git a/requirements-tests.txt b/requirements-tests.txt index 791dfb5..3a40ef0 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,3 +1,4 @@ pep8>=1.7.0,<1.8 flake8>=2.5.1,<2.6 mock>=1.0.1,<1.1 +requests \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/30d27bbc/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 19926ef..dc1589f 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ DOC_TEST_MODULES = ['libcloud.compute.drivers.dummy', SUPPORTED_VERSIONS = ['2.5', '2.6', '2.7', 'PyPy', '3.x'] TEST_REQUIREMENTS = [ - 'mock' + 'mock', + 'requests' ] if PY2_pre_279 or PY3_pre_32:
