incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [33/41] git commit: [#6656] ticket:437 Check rate limit values and sleep until reset
Date Thu, 14 Nov 2013 15:37:25 GMT
[#6656] ticket:437 Check rate limit values and sleep until reset


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/4d518a0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/4d518a0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/4d518a0f

Branch: refs/heads/cj/6845
Commit: 4d518a0f2c2e977d46f2cce6b7b81b82e6097516
Parents: d396c9e
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Fri Oct 25 12:53:52 2013 +0300
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000

----------------------------------------------------------------------
 .../forgeimporters/github/__init__.py           | 16 ++++++++--
 .../tests/github/test_extractor.py              | 31 ++++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 5726326..4932697 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -18,6 +18,8 @@
 import re
 import logging
 import json
+import time
+from datetime import datetime
 
 from forgeimporters import base
 
@@ -48,8 +50,18 @@ class GitHubProjectExtractor(base.ProjectExtractor):
         return url
 
     def urlopen(self, url, **kw):
-        url = self.add_token(url)
-        return super(GitHubProjectExtractor, self).urlopen(url, **kw)
+        resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+        remain = resp.info().get('X-RateLimit-Remaining')
+        if remain and int(remain) == 0:
+            reset = resp.info().get('X-RateLimit-Reset')
+            limit = resp.info().get('X-RateLimit-Limit')
+            reset = datetime.utcfromtimestamp(int(reset))
+            now = datetime.utcnow()
+            log.warn('Rate limit exceeded (%s requests/hour). '
+                     'Sleeping until %s UTC' % (limit, reset))
+            time.sleep((reset - now).total_seconds())
+            self.urlopen(url, **kw)
+        return resp
 
     def get_next_page_url(self, link):
         if not link:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index a26438a..b3ad2f2 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,6 +16,7 @@
 #       under the License.
 
 import json
+from datetime import datetime
 from unittest import TestCase
 
 from mock import patch, Mock
@@ -149,3 +150,33 @@ class TestGitHubProjectExtractor(TestCase):
         e.urlopen(url)
         request = urlopen.call_args[0][0]
         self.assertEqual(request.get_full_url(), url + '&access_token=abc')
+
+    @patch('forgeimporters.base.h.urlopen')
+    @patch('forgeimporters.github.time.sleep')
+    @patch('forgeimporters.github.log')
+    def test_urlopen_rate_limit(self, log, sleep, urlopen):
+        limit_exceeded_headers = {
+            'X-RateLimit-Limit': '10',
+            'X-RateLimit-Remaining': '0',
+            'X-RateLimit-Reset': '1382693522',
+        }
+        response_limit_exceeded = StringIO('{}')
+        response_limit_exceeded.info = lambda: limit_exceeded_headers
+        response_ok = StringIO('{}')
+        response_ok.info = lambda: {}
+        results = [response_limit_exceeded, response_ok]
+        urlopen.side_effect = lambda *a, **kw: results.pop(0)
+        e = github.GitHubProjectExtractor('test_project')
+        e.get_page('fake')
+        self.assertEqual(sleep.call_count, 1)
+        self.assertEqual(urlopen.call_count, 2)
+        log.warn.assert_called_once_with(
+            'Rate limit exceeded (10 requests/hour). '
+            'Sleeping until 2013-10-25 09:32:02 UTC'
+        )
+        sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
+        urlopen.side_effect = lambda *a, **kw: response_ok
+        e.get_page('fake 2')
+        self.assertEqual(sleep.call_count, 0)
+        self.assertEqual(urlopen.call_count, 1)
+        self.assertEqual(log.warn.call_count, 0)


Mime
View raw message