allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [04/18] git commit: [#6994] Run individual tests in parallel
Date Thu, 09 Jan 2014 20:36:51 GMT
[#6994] Run individual tests in parallel

Signed-off-by: Tim Van Steenburgh <tvansteenburgh@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d4c06e8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d4c06e8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d4c06e8c

Branch: refs/heads/tv/6905
Commit: d4c06e8c5e4e205bd29907bd126b147e5ad6ddf5
Parents: 8b15d49
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Fri Dec 20 04:41:32 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Tue Jan 7 17:59:45 2014 +0000

----------------------------------------------------------------------
 Allura/allura/tests/__init__.py   |  4 ++++
 Allura/allura/tests/test_utils.py |  1 +
 requirements-common.txt           |  2 +-
 run_tests                         | 38 +++++++++++++++++++++++++++++-----
 4 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/__init__.py b/Allura/allura/tests/__init__.py
index 3fb7e93..c715a48 100644
--- a/Allura/allura/tests/__init__.py
+++ b/Allura/allura/tests/__init__.py
@@ -21,6 +21,10 @@
 
 import alluratest.controller
 
+# HACK: prevents test suite from crashing when running under the nose
+#       MultiProcessing plugin
+import socket
+socket.setdefaulttimeout(None)
 
 class TestController(alluratest.controller.TestController):
     """

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/test_utils.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 494e893..b8b5de7 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -94,6 +94,7 @@ class TestAntispam(unittest.TestCase):
 
     def setUp(self):
         setup_unit_test()
+        pylons.request._push_object(Request.blank('/'))
         pylons.request.remote_addr = '127.0.0.1'
         self.a = utils.AntiSpam()
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/requirements-common.txt
----------------------------------------------------------------------
diff --git a/requirements-common.txt b/requirements-common.txt
index 7f48285..5ffd19f 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -74,7 +74,7 @@ smmap==0.8.1
 datadiff==1.1.5
 ipython==0.11
 mock==1.0.1
-nose==1.1.2
+nose==1.3.0
 pyflakes==0.5.0
 WebTest==1.4.0
 clonedigger==1.1.0

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index a85aefe..38e63e4 100755
--- a/run_tests
+++ b/run_tests
@@ -20,12 +20,26 @@
 import argparse
 from copy import copy
 from glob import glob
+import multiprocessing
 from multiprocessing.pool import ThreadPool
 import subprocess
 import sys
 import threading
 import textwrap
 
+CPUS = multiprocessing.cpu_count()
+CONCURRENT_SUITES = (CPUS // 4) or CPUS
+CONCURRENT_TESTS = (CPUS // CONCURRENT_SUITES) or 1
+PROC_TIMEOUT = 120
+
+ALT_PKG_PATHS = {
+        'Allura': 'allura/tests/',
+        }
+
+NOT_MULTIPROC_SAFE = [
+        'ForgeGit',
+        'ForgeSVN',
+        ]
 
 def run_one(cmd, **popen_kwargs):
     print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
@@ -76,7 +90,7 @@ def run_many(cmds, processes=None):
 
 
 def get_packages():
-    packages = [p.split('/')[0] for p in glob("*/setup.py")]
+    packages = sorted([p.split('/')[0] for p in glob("*/setup.py")])
 
     # make it first, to catch syntax errors
     packages.remove('AlluraTest')
@@ -95,16 +109,26 @@ def check_packages(packages):
 
 
 def run_tests_in_parallel(options, nosetests_args):
+    def get_pkg_path(pkg):
+        return ALT_PKG_PATHS.get(pkg, '')
+    def get_multiproc_args(pkg):
+        return ('--processes={procs_per_suite} --process-timeout={proc_timeout}'.format(
+                    procs_per_suite=options.concurrent_tests,
+                    proc_timeout=PROC_TIMEOUT)
+                if pkg not in NOT_MULTIPROC_SAFE else '')
+
     cmds = []
     for package in check_packages(options.packages):
         cover_package = package.lower()
         our_nosetests_args = copy(nosetests_args)
         our_nosetests_args.append('--cover-package={}'.format(cover_package))
-        cmd = "nosetests {nosetests_args}".format(
+        cmd = "nosetests {pkg_path} {nosetests_args} {multiproc_args}".format(
+            pkg_path=get_pkg_path(package),
             nosetests_args=' '.join(our_nosetests_args),
+            multiproc_args=get_multiproc_args(package),
         )
         cmds.append((cmd, dict(cwd=package)))
-    return run_many(cmds, processes=options.num_processes)
+    return run_many(cmds, processes=options.concurrent_suites)
 
 
 def parse_args():
@@ -113,8 +137,12 @@ def parse_args():
                                         All additional arguments are passed along to nosetests
                                           (e.g. -v --with-coverage)
                                         Note: --cover-package will be set automatically to
the appropriate value'''))
-    parser.add_argument('-n', help='Number of processes to use at once. Default: # CPUs',
-                        dest='num_processes', type=int, default=None)
+    parser.add_argument('-n', help='Number of test suites to run concurrently in separate
'
+                                   'processes. Default: # CPUs / 4',
+                        dest='concurrent_suites', type=int, default=CONCURRENT_SUITES)
+    parser.add_argument('-m', help='Number of tests to run concurrently in separate '
+                                   'processes, per suite. Default: # CPUs / # concurrent
suites',
+                        dest='concurrent_tests', type=int, default=CONCURRENT_TESTS)
     parser.add_argument('-p', help='List of packages to run tests on. Default: all',
                         dest='packages', choices=get_packages(), default=get_packages(),
                         nargs='+')


Mime
View raw message