libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject svn commit: r1209961 - in /libcloud/branches/py3k: libcloud/common/ libcloud/storage/ libcloud/storage/drivers/ test/ test/storage/
Date Sat, 03 Dec 2011 18:12:29 GMT
Author: tomaz
Date: Sat Dec  3 18:12:28 2011
New Revision: 1209961

URL: http://svn.apache.org/viewvc?rev=1209961&view=rev
Log:
Storage tests now pass with Python 3.

Modified:
    libcloud/branches/py3k/libcloud/common/base.py
    libcloud/branches/py3k/libcloud/storage/base.py
    libcloud/branches/py3k/libcloud/storage/drivers/atmos.py
    libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py
    libcloud/branches/py3k/libcloud/storage/drivers/dummy.py
    libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py
    libcloud/branches/py3k/libcloud/storage/drivers/s3.py
    libcloud/branches/py3k/test/__init__.py
    libcloud/branches/py3k/test/storage/test_atmos.py
    libcloud/branches/py3k/test/storage/test_base.py
    libcloud/branches/py3k/test/storage/test_cloudfiles.py
    libcloud/branches/py3k/test/storage/test_google_storage.py
    libcloud/branches/py3k/test/storage/test_s3.py

Modified: libcloud/branches/py3k/libcloud/common/base.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/common/base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/common/base.py (original)
+++ libcloud/branches/py3k/libcloud/common/base.py Sat Dec  3 18:12:28 2011
@@ -30,6 +30,8 @@ from libcloud.py3 import urllib
 from libcloud.py3 import httplib
 from libcloud.py3 import urlparse
 from libcloud.py3 import StringIO
+from libcloud.py3 import u
+
 from libcloud.common.types import LibcloudError, MalformedResponseError
 
 from libcloud.httplib_ssl import LibcloudHTTPSConnection

Modified: libcloud/branches/py3k/libcloud/storage/base.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/base.py (original)
+++ libcloud/branches/py3k/libcloud/storage/base.py Sat Dec  3 18:12:28 2011
@@ -20,11 +20,14 @@ Provides base classes for working with s
 # Backward compatibility for Python 2.5
 from __future__ import with_statement
 
-from libcloud.py3 import httplib
 import os.path                          # pylint: disable-msg=W0404
 import hashlib
 from os.path import join as pjoin
 
+from libcloud.py3 import httplib
+from libcloud.py3 import next
+from libcloud.py3 import b
+
 from libcloud import utils
 from libcloud.common.types import LibcloudError
 from libcloud.common.base import ConnectionUserAndKey, BaseDriver
@@ -466,7 +469,7 @@ class StorageDriver(BaseDriver):
         stream = utils.read_in_chunks(response, chunk_size)
 
         try:
-            data_read = stream.next()
+            data_read = next(stream)
         except StopIteration:
             # Empty response?
             return False
@@ -475,11 +478,11 @@ class StorageDriver(BaseDriver):
 
         with open(file_path, 'wb') as file_handle:
             while len(data_read) > 0:
-                file_handle.write(data_read)
+                file_handle.write(b(data_read))
                 bytes_transferred += len(data_read)
 
                 try:
-                    data_read = stream.next()
+                    data_read = next(stream)
                 except StopIteration:
                     data_read = ''
 
@@ -505,9 +508,10 @@ class StorageDriver(BaseDriver):
         headers = headers or {}
 
         if file_path and not os.path.exists(file_path):
-          raise OSError('File %s does not exist' % (file_path))
+            raise OSError('File %s does not exist' % (file_path))
 
-        if iterator is not None and not hasattr(iterator, 'next'):
+        if iterator is not None and not hasattr(iterator, 'next') and not \
+           hasattr(iterator, '__next__'):
             raise AttributeError('iterator object must implement next() ' +
                                  'method.')
 
@@ -584,7 +588,7 @@ class StorageDriver(BaseDriver):
 
         if calculate_hash:
             data_hash = self._get_hash_function()
-            data_hash.update(data)
+            data_hash.update(b(data))
 
         try:
             response.connection.connection.send(data)
@@ -639,7 +643,7 @@ class StorageDriver(BaseDriver):
 
         bytes_transferred = 0
         try:
-            chunk = generator.next()
+            chunk = next(generator)
         except StopIteration:
             # Special case when StopIteration is thrown on the first iteration -
             # create a 0-byte long object
@@ -670,10 +674,10 @@ class StorageDriver(BaseDriver):
 
             bytes_transferred += len(chunk)
             if calculate_hash:
-                data_hash.update(chunk)
+                data_hash.update(b(chunk))
 
             try:
-                chunk = generator.next()
+                chunk = next(generator)
             except StopIteration:
                 chunk = ''
 

Modified: libcloud/branches/py3k/libcloud/storage/drivers/atmos.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/atmos.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/atmos.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/atmos.py Sat Dec  3 18:12:28 2011
@@ -13,15 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import base64
 import hashlib
 import hmac
-from libcloud.py3 import httplib
 import time
+
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
 from libcloud.py3 import urllib
-import urlparse
+from libcloud.py3 import urlparse
+from libcloud.py3 import next
+from libcloud.py3 import b
 
-from xml.etree import ElementTree
+if PY3:
+    from io import FileIO as file
 
 from libcloud import utils
 from libcloud.common.base import ConnectionUserAndKey, XmlResponse
@@ -100,8 +106,8 @@ class AtmosConnection(ConnectionUserAndK
         signature.extend([k + ':' + collapse(v) for k, v in xhdrs])
         signature = '\n'.join(signature)
         key = base64.b64decode(self.key)
-        signature = hmac.new(key, signature, hashlib.sha1).digest()
-        return base64.b64encode(signature)
+        signature = hmac.new(b(key), b(signature), hashlib.sha1).digest()
+        return base64.b64encode(b(signature))
 
 class AtmosDriver(StorageDriver):
     connectionCls = AtmosConnection
@@ -132,7 +138,8 @@ class AtmosDriver(StorageDriver):
         path = self._namespace_path(container_name + '/?metadata/system')
         try:
             result = self.connection.request(path)
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code != 1003:
                 raise
             raise ContainerDoesNotExistError(e, self, container_name)
@@ -146,7 +153,8 @@ class AtmosDriver(StorageDriver):
         path = self._namespace_path(container_name + '/')
         try:
             result = self.connection.request(path, method='POST')
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code != 1016:
                 raise
             raise ContainerAlreadyExistsError(e, self, container_name)
@@ -156,7 +164,8 @@ class AtmosDriver(StorageDriver):
         try:
             self.connection.request(self._namespace_path(container.name + '/'),
                                     method='DELETE')
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code == 1003:
                 raise ContainerDoesNotExistError(e, self, container.name)
             elif e.code == 1023:
@@ -174,7 +183,8 @@ class AtmosDriver(StorageDriver):
 
             result = self.connection.request(path + '?metadata/user')
             user_meta = self._emc_meta(result)
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code != 1003:
                 raise
             raise ObjectDoesNotExistError(e, self, object_name)
@@ -204,7 +214,8 @@ class AtmosDriver(StorageDriver):
 
         try:
             self.connection.request(request_path + '?metadata/system')
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code != 1003:
                 raise
             method = 'POST'
@@ -248,7 +259,7 @@ class AtmosDriver(StorageDriver):
         generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True)
         bytes_transferred = 0
         try:
-            chunk = generator.next()
+            chunk = next(generator)
         except StopIteration:
             chunk = ''
 
@@ -256,7 +267,7 @@ class AtmosDriver(StorageDriver):
 
         while True:
             end = bytes_transferred + len(chunk) - 1
-            data_hash.update(chunk)
+            data_hash.update(b(chunk))
             headers = {
                 'x-emc-meta': 'md5=' + data_hash.hexdigest(),
             }
@@ -267,7 +278,7 @@ class AtmosDriver(StorageDriver):
             bytes_transferred += len(chunk)
 
             try:
-                chunk = generator.next()
+                chunk = next(generator)
             except StopIteration:
                 break
             if len(chunk) == 0:
@@ -327,7 +338,8 @@ class AtmosDriver(StorageDriver):
         path = self._namespace_path(obj.container.name + '/' + obj.name)
         try:
             self.connection.request(path, method='DELETE')
-        except AtmosError, e:
+        except AtmosError:
+            e = sys.exc_info()[1]
             if e.code != 1003:
                 raise
             raise ObjectDoesNotExistError(e, self, obj.name)

Modified: libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py Sat Dec  3 18:12:28 2011
@@ -21,6 +21,12 @@ try:
 except ImportError:
     import json
 
+from libcloud.py3 import PY3
+
+if PY3:
+    from io import FileIO as file
+
+
 from libcloud.utils import read_in_chunks
 from libcloud.common.types import MalformedResponseError, LibcloudError
 from libcloud.common.base import Response, RawResponse

Modified: libcloud/branches/py3k/libcloud/storage/drivers/dummy.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/dummy.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/dummy.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/dummy.py Sat Dec  3 18:12:28 2011
@@ -17,6 +17,12 @@ import os.path
 import random
 import hashlib
 
+from libcloud.py3 import PY3
+from libcloud.py3 import b
+
+if PY3:
+    from io import FileIO as file
+
 from libcloud.common.types import LibcloudError
 
 from libcloud.storage.base import Object, Container, StorageDriver
@@ -61,10 +67,14 @@ class DummyIterator(object):
             raise StopIteration
 
         value = self._data[self._current_item]
-        self.hash.update(value)
+        self.hash.update(b(value))
         self._current_item += 1
         return value
 
+    def __next__(self):
+        return self.next()
+
+
 class DummyStorageDriver(StorageDriver):
     """
     Dummy Storage driver.
@@ -318,7 +328,7 @@ class DummyStorageDriver(StorageDriver):
         ...    iterator=DummyFileObject(5, 10), extra={})
         >>> stream = container.download_object_as_stream(obj)
         >>> stream #doctest: +ELLIPSIS
-        <closed file ..., mode '<uninitialized file>' at 0x...>
+        <...closed...>
         """
 
         return DummyFileObject()

Modified: libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py Sat Dec  3 18:12:28
2011
@@ -14,8 +14,6 @@
 # limitations under the License.
 
 import time
-from libcloud.py3 import httplib
-from libcloud.py3 import urllib
 import copy
 import base64
 import hmac
@@ -23,6 +21,8 @@ import hmac
 from hashlib import sha1
 from email.utils import formatdate
 
+from libcloud.py3 import b
+
 from libcloud.common.base import ConnectionUserAndKey
 
 from libcloud.storage.drivers.s3 import S3StorageDriver, S3Response
@@ -122,7 +122,7 @@ class GoogleStorageConnection(Connection
 
         string_to_sign = '\n'.join(values_to_sign)
         b64_hmac = base64.b64encode(
-            hmac.new(secret_key, string_to_sign, digestmod=sha1).digest()
+            hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
         )
         return b64_hmac
 

Modified: libcloud/branches/py3k/libcloud/storage/drivers/s3.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/s3.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/s3.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/s3.py Sat Dec  3 18:12:28 2011
@@ -14,8 +14,6 @@
 # limitations under the License.
 
 import time
-from libcloud.py3 import httplib
-from libcloud.py3 import urllib
 import copy
 import base64
 import hmac
@@ -23,6 +21,11 @@ import hmac
 from hashlib import sha1
 from xml.etree.ElementTree import Element, SubElement, tostring
 
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
+from libcloud.py3 import urllib
+from libcloud.py3 import b
+
 from libcloud.utils import fixxpath, findtext, in_development_warning
 from libcloud.utils import read_in_chunks
 from libcloud.common.types import InvalidCredsError, LibcloudError
@@ -154,7 +157,7 @@ class S3Connection(ConnectionUserAndKey)
 
         string_to_sign = '\n'.join(values_to_sign)
         b64_hmac = base64.b64encode(
-            hmac.new(secret_key, string_to_sign, digestmod=sha1).digest()
+            hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
         )
         return b64_hmac
 
@@ -214,7 +217,12 @@ class S3StorageDriver(StorageDriver):
             root = Element('CreateBucketConfiguration')
             child = SubElement(root, 'LocationConstraint')
             child.text = self.ex_location_name
-            data = tostring(root)
+
+            if PY3:
+                encoding = 'unicode'
+            else:
+                encoding = None
+            data = tostring(root, encoding=encoding)
         else:
             data = ''
 

Modified: libcloud/branches/py3k/test/__init__.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/__init__.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/__init__.py (original)
+++ libcloud/branches/py3k/test/__init__.py Sat Dec  3 18:12:28 2011
@@ -21,6 +21,8 @@ from cgi import parse_qs
 from libcloud.py3 import httplib
 from libcloud.py3 import StringIO
 from libcloud.py3 import urlparse
+from libcloud.py3 import u
+
 
 XML_HEADERS = {'content-type': 'application/xml'}
 
@@ -76,7 +78,7 @@ class MockResponse(object):
 
     def __init__(self, status, body, headers=None, reason=None):
         self.status = status
-        self.body = StringIO(body)
+        self.body = StringIO(u(body))
         self.headers = headers or self.headers
         self.reason = reason or self.reason
 
@@ -241,6 +243,9 @@ class MockRawResponse(BaseMockHttpObject
         self._current_item += 1
         return value
 
+    def __next__(self):
+        return self.next()
+
     def _generate_random_data(self, size):
         data = []
         current_size = 0

Modified: libcloud/branches/py3k/test/storage/test_atmos.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_atmos.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_atmos.py (original)
+++ libcloud/branches/py3k/test/storage/test_atmos.py Sat Dec  3 18:12:28 2011
@@ -14,13 +14,13 @@
 # limitations under the License.
 
 import base64
-import httplib
 import os.path
 import sys
 import unittest
-import urlparse
 
-from xml.etree import ElementTree
+from libcloud.py3 import httplib
+from libcloud.py3 import urlparse
+from libcloud.py3 import b
 
 import libcloud.utils
 
@@ -44,7 +44,7 @@ class AtmosTests(unittest.TestCase):
         AtmosMockHttp.type = None
         AtmosMockHttp.upload_created = False
         AtmosMockRawResponse.type = None
-        self.driver = AtmosDriver('dummy', base64.b64encode('dummy'))
+        self.driver = AtmosDriver('dummy', base64.b64encode(b('dummy')))
         self._remove_test_file()
 
     def tearDown(self):
@@ -342,7 +342,7 @@ class AtmosTests(unittest.TestCase):
 
     def test_signature_algorithm(self):
         test_uid = 'fredsmagicuid'
-        test_key = base64.b64encode('ssssshhhhhmysecretkey')
+        test_key = base64.b64encode(b('ssssshhhhhmysecretkey'))
         test_date = 'Mon, 04 Jul 2011 07:39:19 GMT'
         test_values = [
             ('GET', '/rest/namespace/foo', '', {},
@@ -372,7 +372,7 @@ class AtmosTests(unittest.TestCase):
             c.driver = d
             headers = c.add_default_headers(headers)
             headers['Date'] = headers['x-emc-date'] = test_date
-            self.assertEqual(c._calculate_signature({}, headers), expected)
+            self.assertEqual(c._calculate_signature({}, headers), b(expected))
 
 class AtmosMockHttp(StorageMockHttp, unittest.TestCase):
     fixtures = StorageFileFixtures('atmos')

Modified: libcloud/branches/py3k/test/storage/test_base.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_base.py (original)
+++ libcloud/branches/py3k/test/storage/test_base.py Sat Dec  3 18:12:28 2011
@@ -17,9 +17,15 @@ import sys
 import unittest
 import hashlib
 
-from StringIO import StringIO
 from mock import Mock
 
+from libcloud.py3 import StringIO
+from libcloud.py3 import PY3
+from libcloud.py3 import b
+
+if PY3:
+    from io import FileIO as file
+
 from libcloud.storage.base import StorageDriver
 
 from test import StorageMockHttp # pylint: disable-msg=E0611
@@ -73,7 +79,12 @@ class BaseStorageTests(unittest.TestCase
 
     def test_upload_zero_bytes_long_object_via_stream(self):
         iterator = Mock()
-        iterator.next.side_effect = StopIteration()
+
+        if PY3:
+            iterator.__next__ = Mock()
+            iterator.__next__.side_effect = StopIteration()
+        else:
+            iterator.next.side_effect = StopIteration()
 
         def mock_send(data):
             self.send_called += 1
@@ -88,7 +99,7 @@ class BaseStorageTests(unittest.TestCase
                                            chunked=False, calculate_hash=True)
 
         self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5('').hexdigest())
+        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
         self.assertEqual(bytes_transferred, 0)
         self.assertEqual(self.send_called, 1)
 
@@ -99,7 +110,7 @@ class BaseStorageTests(unittest.TestCase
                                            chunked=True, calculate_hash=True)
 
         self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5('').hexdigest())
+        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
         self.assertEqual(bytes_transferred, 0)
         self.assertEqual(self.send_called, 5)
 
@@ -116,7 +127,7 @@ class BaseStorageTests(unittest.TestCase
                                            calculate_hash=True)
 
         self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5(data).hexdigest())
+        self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
         self.assertEqual(bytes_transferred, (len(data)))
         self.assertEqual(self.send_called, 1)
 

Modified: libcloud/branches/py3k/test/storage/test_cloudfiles.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_cloudfiles.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_cloudfiles.py (original)
+++ libcloud/branches/py3k/test/storage/test_cloudfiles.py Sat Dec  3 18:12:28 2011
@@ -17,10 +17,15 @@ import os.path                          
 import sys
 import copy
 import unittest
-import httplib
 
 import libcloud.utils
 
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
+
+if PY3:
+    from io import FileIO as file
+
 from libcloud.common.types import LibcloudError, MalformedResponseError
 from libcloud.storage.base import Container, Object
 from libcloud.storage.types import ContainerAlreadyExistsError

Modified: libcloud/branches/py3k/test/storage/test_google_storage.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_google_storage.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_google_storage.py (original)
+++ libcloud/branches/py3k/test/storage/test_google_storage.py Sat Dec  3 18:12:28 2011
@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import sys
-import httplib
 import unittest
 
 from libcloud.storage.drivers.google_storage import GoogleStorageDriver

Modified: libcloud/branches/py3k/test/storage/test_s3.py
URL: http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_s3.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_s3.py (original)
+++ libcloud/branches/py3k/test/storage/test_s3.py Sat Dec  3 18:12:28 2011
@@ -15,9 +15,10 @@
 
 import os
 import sys
-import httplib
 import unittest
 
+from libcloud.py3 import httplib
+
 from libcloud.common.types import InvalidCredsError
 from libcloud.common.types import LibcloudError
 from libcloud.storage.base import Container, Object
@@ -276,7 +277,8 @@ class S3Tests(unittest.TestCase):
         self.mock_response_klass.type = 'UNAUTHORIZED'
         try:
             self.driver.list_containers()
-        except InvalidCredsError, e:
+        except InvalidCredsError:
+            e = sys.exc_info()[1]
             self.assertEqual(True, isinstance(e, InvalidCredsError))
         else:
             self.fail('Exception was not thrown')
@@ -514,7 +516,8 @@ class S3Tests(unittest.TestCase):
                                       object_name=object_name,
                                       verify_hash=True,
                                       ex_storage_class='invalid-class')
-        except ValueError, e:
+        except ValueError:
+            e = sys.exc_info()[1]
             self.assertTrue(str(e).lower().find('invalid storage class') != -1)
         else:
             self.fail('Exception was not thrown')



Mime
View raw message