allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject allura git commit: Avoid importer requests hanging indefinitely
Date Thu, 15 Dec 2016 15:27:21 GMT
Repository: allura
Updated Branches:
  refs/heads/master 7eee12cf8 -> 00fb5b4b8


Avoid importer requests hanging indefinitely

* add a long timeout to the main urlopen() in case some requests
  do take a very long time
* add a short timeout to the HEAD and github requests


Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/00fb5b4b
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/00fb5b4b
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/00fb5b4b

Branch: refs/heads/master
Commit: 00fb5b4b865c748f886bed237e658a31dfe2f4e6
Parents: 7eee12c
Author: Dave Brondsema <dave@brondsema.net>
Authored: Wed Dec 14 14:04:43 2016 -0500
Committer: Dave Brondsema <dave@brondsema.net>
Committed: Thu Dec 15 10:26:32 2016 -0500

----------------------------------------------------------------------
 ForgeImporters/forgeimporters/base.py                         | 4 ++--
 ForgeImporters/forgeimporters/github/__init__.py              | 7 ++++---
 ForgeImporters/forgeimporters/github/tests/test_oauth.py      | 2 +-
 .../forgeimporters/tests/github/functional/test_github.py     | 3 ++-
 ForgeImporters/forgeimporters/tests/test_base.py              | 2 +-
 ForgeImporters/forgeimporters/trac/__init__.py                | 2 +-
 6 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 22bd27c..fb2fa21 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -166,11 +166,11 @@ class ProjectExtractor(object):
             self.get_page(page_name, **kw)
 
     @staticmethod
-    def urlopen(url, retries=3, codes=(408, 500, 502, 503, 504), **kw):
+    def urlopen(url, retries=3, codes=(408, 500, 502, 503, 504), timeout=120, **kw):
         req = urllib2.Request(url, **kw)
         req.add_header(
             'User-Agent', 'Allura Data Importer (https://allura.apache.org/)')
-        return h.urlopen(req, retries=retries, codes=codes)
+        return h.urlopen(req, retries=retries, codes=codes, timeout=timeout)
 
     def get_page(self, page_name_or_url, parser=None, **kw):
         """Return a Beautiful soup object for the given page name or url.

http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 777c8d0..1dc4ca1 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -107,7 +107,7 @@ class GitHubProjectExtractor(base.ProjectExtractor):
         return resp
 
     def check_readable(self):
-        resp = requests.head(self.add_token(self.get_page_url('project_info')))
+        resp = requests.head(self.add_token(self.get_page_url('project_info')), timeout=10)
         return resp.status_code == 200
 
     def get_next_page_url(self, link):
@@ -182,7 +182,8 @@ def valid_access_token(access_token):
     client_id = config['github_importer.client_id']
     secret = config['github_importer.client_secret']
     token_valid_resp = requests.get('https://api.github.com/applications/{}/tokens/{}'.format(client_id,
access_token),
-                                    auth=requests.auth.HTTPBasicAuth(client_id, secret))
+                                    auth=requests.auth.HTTPBasicAuth(client_id, secret),
+                                    timeout=10)
     return token_valid_resp.status_code == 200
 
 
@@ -239,7 +240,7 @@ class GitHubOAuthMixin(object):
         if not token:
             return False
         url = 'https://api.github.com/?access_token={}'.format(token)
-        r = requests.head(url)
+        r = requests.head(url, timeout=10)
         scopes = r.headers.get('X-OAuth-Scopes', '')
         scopes = [s.strip() for s in scopes.split(',')]
         return scope in scopes

http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/github/tests/test_oauth.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_oauth.py b/ForgeImporters/forgeimporters/github/tests/test_oauth.py
index b336c7a..0bf8f4e 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_oauth.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_oauth.py
@@ -45,7 +45,7 @@ class TestGitHubOAuthMixin(TestController, TestCase):
     def test_oauth_has_access_no_headers(self, req):
         c.user.get_tool_data.return_value = 'token'
         self.assertFalse(self.mix.oauth_has_access('write:repo_hook'))
-        req.head.assert_called_once_with('https://api.github.com/?access_token=token')
+        req.head.assert_called_once_with('https://api.github.com/?access_token=token', timeout=10)
 
     @patch('forgeimporters.github.requests')
     def test_oauth_has_access_with_headers(self, req):

http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
index eb38671..c4d2bce 100644
--- a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
+++ b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
@@ -89,4 +89,5 @@ class TestGitHubOAuth(TestController):
         assert_equal(r.status_int, 200)
 
         valid_access_token_get.assert_called_once_with('https://api.github.com/applications/client_id/tokens/abc',
-                                                       auth=requests.auth.HTTPBasicAuth('client_id',
'secret'))
+                                                       auth=requests.auth.HTTPBasicAuth('client_id',
'secret'),
+                                                       timeout=10)

http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index 5f214ac..f7eb706 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -41,7 +41,7 @@ class TestProjectExtractor(TestCase):
         req = Request.return_value
         req.add_header.assert_called_once_with(
             'User-Agent', 'Allura Data Importer (https://allura.apache.org/)')
-        urlopen.assert_called_once_with(req, retries=3, codes=(408, 500, 502, 503, 504))
+        urlopen.assert_called_once_with(req, retries=3, codes=(408, 500, 502, 503, 504),
timeout=120)
         self.assertEqual(r, urlopen.return_value)
 
 

http://git-wip-us.apache.org/repos/asf/allura/blob/00fb5b4b/ForgeImporters/forgeimporters/trac/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/__init__.py b/ForgeImporters/forgeimporters/trac/__init__.py
index 12e677a..fa733a6 100644
--- a/ForgeImporters/forgeimporters/trac/__init__.py
+++ b/ForgeImporters/forgeimporters/trac/__init__.py
@@ -41,7 +41,7 @@ class TracURLValidator(validators.URL):
         value = value.rstrip('/') + '/'
 
         try:
-            resp = requests.head(value, allow_redirects=True)
+            resp = requests.head(value, allow_redirects=True, timeout=10)
         except IOError:
             # fall through to 'raise' below
             pass


Mime
View raw message