libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject git commit: Use lxml library (if available) for parsing XML. This should substantially reduce parsing time and memory usage for large XML responses (e.g. retrieving all the available images in the EC2 driver).
Date Mon, 10 Mar 2014 17:58:10 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk 68272288f -> 0619c9257


Use lxml library (if available) for parsing XML. This should substantially
reduce parsing time and memory usage for large XML responses (e.g. retrieving
all the available images in the EC2 driver).


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/0619c925
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/0619c925
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/0619c925

Branch: refs/heads/trunk
Commit: 0619c9257556b52794f3a2f70760abc01798faf5
Parents: 6827228
Author: Tomaz Muraus <tomaz@apache.org>
Authored: Mon Mar 10 18:49:58 2014 +0100
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Mon Mar 10 18:49:58 2014 +0100

----------------------------------------------------------------------
 CHANGES.rst                                | 8 ++++++++
 libcloud/common/aws.py                     | 6 +++++-
 libcloud/common/azure.py                   | 8 ++++++--
 libcloud/common/base.py                    | 6 +++++-
 libcloud/compute/drivers/ec2.py            | 5 ++++-
 libcloud/compute/drivers/libvirt_driver.py | 6 +++++-
 libcloud/compute/drivers/opennebula.py     | 6 +++++-
 libcloud/compute/drivers/openstack.py      | 5 ++++-
 libcloud/compute/drivers/opsource.py       | 7 ++++++-
 libcloud/compute/drivers/vcloud.py         | 6 +++++-
 libcloud/dns/drivers/route53.py            | 6 +++++-
 libcloud/dns/drivers/zerigo.py             | 5 ++++-
 libcloud/storage/drivers/s3.py             | 6 +++++-
 libcloud/test/compute/test_abiquo.py       | 5 ++++-
 libcloud/test/compute/test_vcloud.py       | 6 +++++-
 libcloud/test/storage/test_s3.py           | 6 +++++-
 libcloud/utils/py3.py                      | 6 +++++-
 17 files changed, 86 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index f09acb5..6f0ac85 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -4,6 +4,14 @@ Changelog
 Changes with Apache Libcloud in development
 -------------------------------------------
 
+General
+~~~~~~~
+
+- Use lxml library (if available) for parsing XML. This should substantially
+  reduce parsing time and memory usage for large XML responses (e.g. retrieving
+  all the available images in the EC2 driver).
+  [Andrew Mann]
+
 Compute
 ~~~~~~~
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/aws.py
----------------------------------------------------------------------
diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py
index 124a434..1d3dfb4 100644
--- a/libcloud/common/aws.py
+++ b/libcloud/common/aws.py
@@ -17,7 +17,11 @@ import base64
 import hmac
 import time
 from hashlib import sha256
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.common.base import ConnectionUserAndKey, XmlResponse, BaseDriver
 from libcloud.common.types import InvalidCredsError, MalformedResponseError

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/azure.py
----------------------------------------------------------------------
diff --git a/libcloud/common/azure.py b/libcloud/common/azure.py
index d9e81a4..104cca8 100644
--- a/libcloud/common/azure.py
+++ b/libcloud/common/azure.py
@@ -23,7 +23,11 @@ from hashlib import sha256
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import b
 from libcloud.utils.xml import fixxpath
-from xml.etree import ElementTree
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.common.types import InvalidCredsError
 from libcloud.common.types import LibcloudError, MalformedResponseError
@@ -54,7 +58,7 @@ class AzureResponse(XmlResponse):
             # Some APIs respond with an XML error. Others just dump HTML
             body = self.parse_body()
 
-            if type(body) == ElementTree.Element:
+            if type(body) == ET.Element:
                 code = body.findtext(fixxpath(xpath='Code'))
                 message = body.findtext(fixxpath(xpath='Message'))
                 message = message.split('\n')[0]

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 1639958..178c218 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -20,7 +20,11 @@ import copy
 import binascii
 import time
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
+
 from pipes import quote as pquote
 
 try:

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/ec2.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/ec2.py b/libcloud/compute/drivers/ec2.py
index e7752d2..e2899f1 100644
--- a/libcloud/compute/drivers/ec2.py
+++ b/libcloud/compute/drivers/ec2.py
@@ -23,7 +23,10 @@ import base64
 import copy
 import warnings
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.utils.py3 import b, basestring
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/libvirt_driver.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/libvirt_driver.py b/libcloud/compute/drivers/libvirt_driver.py
index ae20f0f..9b71885 100644
--- a/libcloud/compute/drivers/libvirt_driver.py
+++ b/libcloud/compute/drivers/libvirt_driver.py
@@ -22,7 +22,11 @@ import mimetypes
 
 from os.path import join as pjoin
 from collections import defaultdict
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.compute.base import NodeDriver, Node
 from libcloud.compute.base import NodeState

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/opennebula.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/opennebula.py b/libcloud/compute/drivers/opennebula.py
index 08db5ed..c29833e 100644
--- a/libcloud/compute/drivers/opennebula.py
+++ b/libcloud/compute/drivers/opennebula.py
@@ -22,10 +22,14 @@ OpenNebula.org driver.
 
 __docformat__ = 'epytext'
 
-from xml.etree import ElementTree as ET
 from base64 import b64encode
 import hashlib
 
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
+
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import next
 from libcloud.utils.py3 import b

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/openstack.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py
index 610b8f0..aba37bc 100644
--- a/libcloud/compute/drivers/openstack.py
+++ b/libcloud/compute/drivers/openstack.py
@@ -29,7 +29,10 @@ from libcloud.utils.py3 import b
 from libcloud.utils.py3 import next
 from libcloud.utils.py3 import urlparse
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.common.openstack import OpenStackBaseConnection
 from libcloud.common.openstack import OpenStackDriverMixin

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/opsource.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/opsource.py b/libcloud/compute/drivers/opsource.py
index fc98be3..c38b042 100644
--- a/libcloud/compute/drivers/opsource.py
+++ b/libcloud/compute/drivers/opsource.py
@@ -15,7 +15,12 @@
 """
 Opsource Driver
 """
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
+
 from base64 import b64encode
 
 from libcloud.utils.py3 import httplib

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/compute/drivers/vcloud.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/vcloud.py b/libcloud/compute/drivers/vcloud.py
index 7bf78b4..d18d390 100644
--- a/libcloud/compute/drivers/vcloud.py
+++ b/libcloud/compute/drivers/vcloud.py
@@ -30,7 +30,11 @@ urlparse = urlparse.urlparse
 
 import time
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
+
 from xml.parsers.expat import ExpatError
 
 from libcloud.common.base import XmlResponse, ConnectionUserAndKey

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/dns/drivers/route53.py
----------------------------------------------------------------------
diff --git a/libcloud/dns/drivers/route53.py b/libcloud/dns/drivers/route53.py
index 64f34c0..38abca8 100644
--- a/libcloud/dns/drivers/route53.py
+++ b/libcloud/dns/drivers/route53.py
@@ -24,7 +24,11 @@ import uuid
 from libcloud.utils.py3 import httplib
 
 from hashlib import sha1
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.utils.py3 import b, urlencode
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/dns/drivers/zerigo.py
----------------------------------------------------------------------
diff --git a/libcloud/dns/drivers/zerigo.py b/libcloud/dns/drivers/zerigo.py
index 50e3ed0..892b4fb 100644
--- a/libcloud/dns/drivers/zerigo.py
+++ b/libcloud/dns/drivers/zerigo.py
@@ -24,7 +24,10 @@ import base64
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import b
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.utils.misc import merge_valid_keys, get_new_obj
 from libcloud.utils.xml import findtext, findall

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/storage/drivers/s3.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 0dd6694..69dcafc 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -20,7 +20,11 @@ import hmac
 import sys
 
 from hashlib import sha1
-from xml.etree.ElementTree import Element, SubElement
+
+try:
+    from lxml.etree import Element, SubElement
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement
 
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import urlquote

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/compute/test_abiquo.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_abiquo.py b/libcloud/test/compute/test_abiquo.py
index 1377769..2c5cc5c 100644
--- a/libcloud/test/compute/test_abiquo.py
+++ b/libcloud/test/compute/test_abiquo.py
@@ -18,7 +18,10 @@ Abiquo Test Suite
 import unittest
 import sys
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.utils.py3 import httplib
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/compute/test_vcloud.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_vcloud.py b/libcloud/test/compute/test_vcloud.py
index 787fa9f..b44162a 100644
--- a/libcloud/test/compute/test_vcloud.py
+++ b/libcloud/test/compute/test_vcloud.py
@@ -15,7 +15,11 @@
 
 import sys
 import unittest
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 from libcloud.utils.py3 import httplib, b
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/test/storage/test_s3.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py
index 5276b26..667e57c 100644
--- a/libcloud/test/storage/test_s3.py
+++ b/libcloud/test/storage/test_s3.py
@@ -17,7 +17,11 @@ import os
 import sys
 import unittest
 
-from xml.etree import ElementTree as ET
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
+
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import urlparse
 from libcloud.utils.py3 import parse_qs

http://git-wip-us.apache.org/repos/asf/libcloud/blob/0619c925/libcloud/utils/py3.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py
index 6625c86..797d317 100644
--- a/libcloud/utils/py3.py
+++ b/libcloud/utils/py3.py
@@ -22,7 +22,11 @@ from __future__ import absolute_import
 
 import sys
 import types
-from xml.etree import ElementTree as ET
+
+try:
+    from lxml import etree as ET
+except ImportError:
+    from xml.etree import ElementTree as ET
 
 PY2 = False
 PY25 = False


Mime
View raw message