libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject [1/4] libcloud git commit: Add SSLError to retry decorator exceptions
Date Sun, 10 Jan 2016 15:49:35 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk e3bb83801 -> c626d9820


Add SSLError to retry decorator exceptions

Closes #556

Signed-off-by: Tomaz Muraus <tomaz@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/5733de36
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/5733de36
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/5733de36

Branch: refs/heads/trunk
Commit: 5733de362d198e70ae5c81082f96de83f307d4b2
Parents: e3bb838
Author: Scott Kruger <scott.kruger@rackspace.com>
Authored: Thu Jul 30 10:02:31 2015 -0500
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Sun Jan 10 16:31:52 2016 +0100

----------------------------------------------------------------------
 libcloud/test/common/test_retry_limit.py | 15 +++++
 libcloud/utils/misc.py                   | 83 +++++++++++++++++++--------
 2 files changed, 74 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/5733de36/libcloud/test/common/test_retry_limit.py
----------------------------------------------------------------------
diff --git a/libcloud/test/common/test_retry_limit.py b/libcloud/test/common/test_retry_limit.py
index 72e3710..06eac2a 100644
--- a/libcloud/test/common/test_retry_limit.py
+++ b/libcloud/test/common/test_retry_limit.py
@@ -15,10 +15,12 @@
 
 import socket
 import tempfile
+import ssl
 
 from mock import Mock, patch, MagicMock
 
 from libcloud.utils.py3 import httplib
+from libcloud.utils.misc import TRANSIENT_SSL_ERROR
 from libcloud.common.base import Connection
 from libcloud.common.base import Response
 from libcloud.common.exceptions import RateLimitReachedError
@@ -31,6 +33,7 @@ CONFLICT_RESPONSE_STATUS = [
 SIMPLE_RESPONSE_STATUS = ('HTTP/1.1', 429, 'CONFLICT')
 
 
+@patch('os.environ', {'LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS': True})
 class FailedRequestRetryTestCase(unittest.TestCase):
 
     def _raise_socket_error(self):
@@ -50,6 +53,18 @@ class FailedRequestRetryTestCase(unittest.TestCase):
             except Exception:
                 self.fail('Failed to raise socket exception')
 
+    def test_retry_connection_ssl_error(self):
+        conn = Connection(timeout=1, retry_delay=0.1)
+
+        with patch.object(conn, 'connect', Mock()):
+            with patch.object(conn, 'connection') as connection:
+                connection.request = MagicMock(
+                    __name__='request',
+                    side_effect=ssl.SSLError(TRANSIENT_SSL_ERROR))
+
+                self.assertRaises(ssl.SSLError, conn.request, '/')
+                self.assertGreater(connection.request.call_count, 1)
+
     def test_rate_limit_error(self):
         sock = Mock()
         con = Connection()

http://git-wip-us.apache.org/repos/asf/libcloud/blob/5733de36/libcloud/utils/misc.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/misc.py b/libcloud/utils/misc.py
index 970feaa..1902d23 100644
--- a/libcloud/utils/misc.py
+++ b/libcloud/utils/misc.py
@@ -22,14 +22,25 @@ from libcloud.utils.py3 import httplib
 import socket
 from datetime import datetime, timedelta
 import time
+import ssl
 
 from libcloud.common.exceptions import RateLimitReachedError
 
+
+TRANSIENT_SSL_ERROR = 'The read operation timed out'
+
+
+class TransientSSLError(ssl.SSLError):
+    """Represent transient SSL errors, e.g. timeouts"""
+    pass
+
+
 DEFAULT_TIMEOUT = 30
-DEFAULT_SLEEP = 1
-DEFAULT_BACKCOFF = 1
-EXCEPTION_TYPES = (RateLimitReachedError, socket.error, socket.gaierror,
-                   httplib.NotConnected, httplib.ImproperConnectionState)
+DEFAULT_DELAY = 1
+DEFAULT_BACKOFF = 1
+RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
+                    httplib.NotConnected, httplib.ImproperConnectionState,
+                    TransientSSLError)
 
 __all__ = [
     'find',
@@ -298,10 +309,10 @@ class ReprMixin(object):
         return str(self.__repr__())
 
 
-def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None,
-          timeout=None, backoff=None):
+def retry(retry_exceptions=None, retry_delay=None, timeout=None,
+          backoff=None):
     """
-    Retry method that helps to handle common exception.
+    Retry decorator that helps to handle common transient exceptions.
 
     :param retry_exceptions: types of exceptions to retry on.
     :param retry_delay: retry delay between the attempts.
@@ -313,29 +324,53 @@ def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None,
     retry_request = retry(timeout=1, retry_delay=1, backoff=1)
     retry_request(self.connection.request)()
     """
-    def deco_retry(func):
+    if retry_exceptions is None:
+        retry_exceptions = RETRY_EXCEPTIONS
+    if retry_delay is None:
+        retry_delay = DEFAULT_DELAY
+    if timeout is None:
+        timeout = DEFAULT_TIMEOUT
+    if backoff is None:
+        backoff = DEFAULT_BACKOFF
+
+    timeout = max(timeout, 0)
+
+    def transform_ssl_error(func, *args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ssl.SSLError:
+            exc = sys.exc_info()[1]
+
+            if TRANSIENT_SSL_ERROR in str(exc):
+                raise TransientSSLError(*exc.args)
+
+            raise exc
+
+    def decorator(func):
         @wraps(func)
         def retry_loop(*args, **kwargs):
-            delay = retry_delay
+            current_delay = retry_delay
             end = datetime.now() + timedelta(seconds=timeout)
-            exc_info = None
-            while datetime.now() < end:
+
+            while True:
                 try:
-                    result = func(*args, **kwargs)
-                    return result
+                    return transform_ssl_error(func, *args, **kwargs)
                 except retry_exceptions:
-                    e = sys.exc_info()[1]
+                    exc = sys.exc_info()[1]
 
-                    if isinstance(e, RateLimitReachedError):
-                        time.sleep(e.retry_after)
+                    if isinstance(exc, RateLimitReachedError):
+                        time.sleep(exc.retry_after)
+
+                        # Reset retries if we're told to wait due to rate
+                        # limiting
+                        current_delay = retry_delay
                         end = datetime.now() + timedelta(
-                            seconds=e.retry_after + timeout)
+                            seconds=exc.retry_after + timeout)
+                    elif datetime.now() >= end:
+                        raise
                     else:
-                        exc_info = e
-                        time.sleep(delay)
-                        delay *= backoff
-            if exc_info:
-                raise exc_info
-            return func(*args, **kwargs)
+                        time.sleep(current_delay)
+                        current_delay *= backoff
+
         return retry_loop
-    return deco_retry
+    return decorator


Mime
View raw message