Repository: libcloud Updated Branches: refs/heads/trunk 68272288f -> 0619c9257
Use lxml library (if available) for parsing XML. This should substantially reduce parsing time and memory usage for large XML responses (e.g. retrieving all the available images in the EC2 driver). Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/0619c925 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/0619c925 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/0619c925 Branch: refs/heads/trunk Commit: 0619c9257556b52794f3a2f70760abc01798faf5 Parents: 6827228 Author: Tomaz Muraus <[email protected]> Authored: Mon Mar 10 18:49:58 2014 +0100 Committer: Tomaz Muraus <[email protected]> Committed: Mon Mar 10 18:49:58 2014 +0100 ---------------------------------------------------------------------- CHANGES.rst | 8 ++++++++ libcloud/common/aws.py | 6 +++++- libcloud/common/azure.py | 8 ++++++-- libcloud/common/base.py | 6 +++++- libcloud/compute/drivers/ec2.py | 5 ++++- libcloud/compute/drivers/libvirt_driver.py | 6 +++++- libcloud/compute/drivers/opennebula.py | 6 +++++- libcloud/compute/drivers/openstack.py | 5 ++++- libcloud/compute/drivers/opsource.py | 7 ++++++- libcloud/compute/drivers/vcloud.py | 6 +++++- libcloud/dns/drivers/route53.py | 6 +++++- libcloud/dns/drivers/zerigo.py | 5 ++++- libcloud/storage/drivers/s3.py | 6 +++++- libcloud/test/compute/test_abiquo.py | 5 ++++- libcloud/test/compute/test_vcloud.py | 6 +++++- libcloud/test/storage/test_s3.py | 6 +++++- libcloud/utils/py3.py | 6 +++++- 17 files changed, 86 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index f09acb5..6f0ac85 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,14 @@ Changelog Changes with Apache Libcloud in development ------------------------------------------- +General +~~~~~~~ + +- Use lxml library (if available) for parsing XML. This should substantially + reduce parsing time and memory usage for large XML responses (e.g. retrieving + all the available images in the EC2 driver). + [Andrew Mann] + Compute ~~~~~~~ http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/aws.py ---------------------------------------------------------------------- diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py index 124a434..1d3dfb4 100644 --- a/libcloud/common/aws.py +++ b/libcloud/common/aws.py @@ -17,7 +17,11 @@ import base64 import hmac import time from hashlib import sha256 -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.common.base import ConnectionUserAndKey, XmlResponse, BaseDriver from libcloud.common.types import InvalidCredsError, MalformedResponseError http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/azure.py ---------------------------------------------------------------------- diff --git a/libcloud/common/azure.py b/libcloud/common/azure.py index d9e81a4..104cca8 100644 --- a/libcloud/common/azure.py +++ b/libcloud/common/azure.py @@ -23,7 +23,11 @@ from hashlib import sha256 from libcloud.utils.py3 import httplib from libcloud.utils.py3 import b from libcloud.utils.xml import fixxpath -from xml.etree import ElementTree + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.common.types import InvalidCredsError from libcloud.common.types import LibcloudError, MalformedResponseError @@ -54,7 +58,7 @@ class AzureResponse(XmlResponse): # Some APIs respond with an XML error. Others just dump HTML body = self.parse_body() - if type(body) == ElementTree.Element: + if type(body) == ET.Element: code = body.findtext(fixxpath(xpath='Code')) message = body.findtext(fixxpath(xpath='Message')) message = message.split('\n')[0] http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 1639958..178c218 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -20,7 +20,11 @@ import copy import binascii import time -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + from pipes import quote as pquote try: http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/ec2.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/ec2.py b/libcloud/compute/drivers/ec2.py index e7752d2..e2899f1 100644 --- a/libcloud/compute/drivers/ec2.py +++ b/libcloud/compute/drivers/ec2.py @@ -23,7 +23,10 @@ import base64 import copy import warnings -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.utils.py3 import b, basestring http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/libvirt_driver.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/libvirt_driver.py b/libcloud/compute/drivers/libvirt_driver.py index ae20f0f..9b71885 100644 --- a/libcloud/compute/drivers/libvirt_driver.py +++ b/libcloud/compute/drivers/libvirt_driver.py @@ -22,7 +22,11 @@ import mimetypes from os.path import join as pjoin from collections import defaultdict -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.compute.base import NodeDriver, Node from libcloud.compute.base import NodeState http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/opennebula.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/opennebula.py b/libcloud/compute/drivers/opennebula.py index 08db5ed..c29833e 100644 --- a/libcloud/compute/drivers/opennebula.py +++ b/libcloud/compute/drivers/opennebula.py @@ -22,10 +22,14 @@ OpenNebula.org driver. __docformat__ = 'epytext' -from xml.etree import ElementTree as ET from base64 import b64encode import hashlib +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + from libcloud.utils.py3 import httplib from libcloud.utils.py3 import next from libcloud.utils.py3 import b http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 610b8f0..aba37bc 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -29,7 +29,10 @@ from libcloud.utils.py3 import b from libcloud.utils.py3 import next from libcloud.utils.py3 import urlparse -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.common.openstack import OpenStackBaseConnection from libcloud.common.openstack import OpenStackDriverMixin http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/opsource.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/opsource.py b/libcloud/compute/drivers/opsource.py index fc98be3..c38b042 100644 --- a/libcloud/compute/drivers/opsource.py +++ b/libcloud/compute/drivers/opsource.py @@ -15,7 +15,12 @@ """ Opsource Driver """ -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + from base64 import b64encode from libcloud.utils.py3 import httplib http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/vcloud.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/vcloud.py b/libcloud/compute/drivers/vcloud.py index 7bf78b4..d18d390 100644 --- a/libcloud/compute/drivers/vcloud.py +++ b/libcloud/compute/drivers/vcloud.py @@ -30,7 +30,11 @@ urlparse = urlparse.urlparse import time -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + from xml.parsers.expat import ExpatError from libcloud.common.base import XmlResponse, ConnectionUserAndKey http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/dns/drivers/route53.py ---------------------------------------------------------------------- diff --git a/libcloud/dns/drivers/route53.py b/libcloud/dns/drivers/route53.py index 64f34c0..38abca8 100644 --- a/libcloud/dns/drivers/route53.py +++ b/libcloud/dns/drivers/route53.py @@ -24,7 +24,11 @@ import uuid from libcloud.utils.py3 import httplib from hashlib import sha1 -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.utils.py3 import b, urlencode http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/dns/drivers/zerigo.py ---------------------------------------------------------------------- diff --git a/libcloud/dns/drivers/zerigo.py b/libcloud/dns/drivers/zerigo.py index 50e3ed0..892b4fb 100644 --- a/libcloud/dns/drivers/zerigo.py +++ b/libcloud/dns/drivers/zerigo.py @@ -24,7 +24,10 @@ import base64 from libcloud.utils.py3 import httplib from libcloud.utils.py3 import b -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.utils.misc import merge_valid_keys, get_new_obj from libcloud.utils.xml import findtext, findall http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/storage/drivers/s3.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 0dd6694..69dcafc 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -20,7 +20,11 @@ import hmac import sys from hashlib import sha1 -from xml.etree.ElementTree import Element, SubElement + +try: + from lxml.etree import Element, SubElement +except ImportError: + from xml.etree.ElementTree import Element, SubElement from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/compute/test_abiquo.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_abiquo.py b/libcloud/test/compute/test_abiquo.py index 1377769..2c5cc5c 100644 --- a/libcloud/test/compute/test_abiquo.py +++ b/libcloud/test/compute/test_abiquo.py @@ -18,7 +18,10 @@ Abiquo Test Suite import unittest import sys -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.utils.py3 import httplib http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/compute/test_vcloud.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_vcloud.py b/libcloud/test/compute/test_vcloud.py index 787fa9f..b44162a 100644 --- a/libcloud/test/compute/test_vcloud.py +++ b/libcloud/test/compute/test_vcloud.py @@ -15,7 +15,11 @@ import sys import unittest -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET from libcloud.utils.py3 import httplib, b http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/storage/test_s3.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index 5276b26..667e57c 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -17,7 +17,11 @@ import os import sys import unittest -from xml.etree import ElementTree as ET +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlparse from libcloud.utils.py3 import parse_qs http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/utils/py3.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py index 6625c86..797d317 100644 --- a/libcloud/utils/py3.py +++ b/libcloud/utils/py3.py @@ -22,7 +22,11 @@ from __future__ import absolute_import import sys import types -from xml.etree import ElementTree as ET + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET PY2 = False PY25 = False
