incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [1/8] git commit: [#6656] ticket:437 More careful rate limit checking
Date Wed, 13 Nov 2013 22:44:57 GMT
Updated Branches:
  refs/heads/master 2c6339ab4 -> c1fc7144e


[#6656] ticket:437 More careful rate limit checking


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3c07c836
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3c07c836
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3c07c836

Branch: refs/heads/master
Commit: 3c07c83643389b4402fb2c9b830ce7f835114517
Parents: 4d518a0
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Fri Oct 25 16:14:50 2013 +0300
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000

----------------------------------------------------------------------
 .../forgeimporters/github/__init__.py           | 32 +++++++++++++-----
 .../tests/github/test_extractor.py              | 35 +++++++++++++++++---
 2 files changed, 54 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 4932697..951d670 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -19,6 +19,7 @@ import re
 import logging
 import json
 import time
+import urllib2
 from datetime import datetime
 
 from forgeimporters import base
@@ -49,18 +50,31 @@ class GitHubProjectExtractor(base.ProjectExtractor):
             url += glue + 'access_token=' + self.token
         return url
 
+    def wait_for_limit_reset(self, headers):
+        reset = headers.get('X-RateLimit-Reset')
+        limit = headers.get('X-RateLimit-Limit')
+        reset = datetime.utcfromtimestamp(int(reset))
+        now = datetime.utcnow()
+        log.warn('Rate limit exceeded (%s requests/hour). '
+                 'Sleeping until %s UTC' % (limit, reset))
+        time.sleep((reset - now).total_seconds())
+
     def urlopen(self, url, **kw):
-        resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+        try:
+            resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+        except urllib2.HTTPError as e:
+            # GitHub will return 403 if rate limit exceeded.
+            # We're checking for limit on every request below, but we still
+            # can get 403 if other import task exceeds the limit before.
+            if e.code == 403 and e.info().get('X-RateLimit-Remaining') == '0':
+                self.wait_for_limit_reset(e.info())
+                return self.urlopen(url, **kw)
+            else:
+                raise e
         remain = resp.info().get('X-RateLimit-Remaining')
         if remain and int(remain) == 0:
-            reset = resp.info().get('X-RateLimit-Reset')
-            limit = resp.info().get('X-RateLimit-Limit')
-            reset = datetime.utcfromtimestamp(int(reset))
-            now = datetime.utcnow()
-            log.warn('Rate limit exceeded (%s requests/hour). '
-                     'Sleeping until %s UTC' % (limit, reset))
-            time.sleep((reset - now).total_seconds())
-            self.urlopen(url, **kw)
+            self.wait_for_limit_reset(resp.info())
+            return self.urlopen(url, **kw)
         return resp
 
     def get_next_page_url(self, link):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index b3ad2f2..e6c96e3 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,8 +16,8 @@
 #       under the License.
 
 import json
-from datetime import datetime
 from unittest import TestCase
+import urllib2
 
 from mock import patch, Mock
 
@@ -164,8 +164,7 @@ class TestGitHubProjectExtractor(TestCase):
         response_limit_exceeded.info = lambda: limit_exceeded_headers
         response_ok = StringIO('{}')
         response_ok.info = lambda: {}
-        results = [response_limit_exceeded, response_ok]
-        urlopen.side_effect = lambda *a, **kw: results.pop(0)
+        urlopen.side_effect = [response_limit_exceeded, response_ok]
         e = github.GitHubProjectExtractor('test_project')
         e.get_page('fake')
         self.assertEqual(sleep.call_count, 1)
@@ -175,8 +174,36 @@ class TestGitHubProjectExtractor(TestCase):
             'Sleeping until 2013-10-25 09:32:02 UTC'
         )
         sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
-        urlopen.side_effect = lambda *a, **kw: response_ok
+        response_ok = StringIO('{}')
+        response_ok.info = lambda: {}
+        urlopen.side_effect = [response_ok]
         e.get_page('fake 2')
         self.assertEqual(sleep.call_count, 0)
         self.assertEqual(urlopen.call_count, 1)
         self.assertEqual(log.warn.call_count, 0)
+
+    @patch('forgeimporters.base.h.urlopen')
+    @patch('forgeimporters.github.time.sleep')
+    @patch('forgeimporters.github.log')
+    def test_urlopen_rate_limit_403(self, log, sleep, urlopen):
+        '''Test that urlopen catches 403 which may happen if limit exceeded by another task'''
+        limit_exceeded_headers = {
+            'X-RateLimit-Limit': '10',
+            'X-RateLimit-Remaining': '0',
+            'X-RateLimit-Reset': '1382693522',
+        }
+        def urlopen_side_effect(*a, **kw):
+            mock_resp = StringIO('{}')
+            mock_resp.info = lambda: {}
+            urlopen.side_effect = [mock_resp]
+            raise urllib2.HTTPError(
+                'url', 403, 'msg', limit_exceeded_headers, StringIO('{}'))
+        urlopen.side_effect = urlopen_side_effect
+        e = github.GitHubProjectExtractor('test_project')
+        e.get_page('fake')
+        self.assertEqual(sleep.call_count, 1)
+        self.assertEqual(urlopen.call_count, 2)
+        log.warn.assert_called_once_with(
+            'Rate limit exceeded (10 requests/hour). '
+            'Sleeping until 2013-10-25 09:32:02 UTC'
+        )


Mime
View raw message