kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [1/5] kudu git commit: iwyu: run IWYU on multiple cores
Date Wed, 18 Apr 2018 23:52:03 GMT
Repository: kudu
Updated Branches:
  refs/heads/master c25b7cbd8 -> a9271b05d


iwyu: run IWYU on multiple cores

This changes iwyu_tool to use a number of parallel IWYU processes equal
to the number of cores, resulting in a huge speedup for common dev
workflows.

Change-Id: I97206318943b408c6fdd5637c6ccb79ee4d87d3f
Reviewed-on: http://gerrit.cloudera.org:8080/10104
Reviewed-by: Dan Burkert <danburkert@apache.org>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/63216152
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/63216152
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/63216152

Branch: refs/heads/master
Commit: 6321615289c4dd35eeb0ca85b74bf6be778251aa
Parents: c25b7cb
Author: Todd Lipcon <todd@apache.org>
Authored: Wed Apr 18 12:30:42 2018 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Wed Apr 18 23:50:58 2018 +0000

----------------------------------------------------------------------
 build-support/iwyu/iwyu_tool.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/63216152/build-support/iwyu/iwyu_tool.py
----------------------------------------------------------------------
diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py
index 8943226..80bea51 100755
--- a/build-support/iwyu/iwyu_tool.py
+++ b/build-support/iwyu/iwyu_tool.py
@@ -70,12 +70,14 @@ Example usage with CMake:
 See iwyu_tool.py -h for more details on command-line arguments.
 """
 
-import os
-import sys
-import json
 import argparse
-import subprocess
+import json
+import multiprocessing
+from multiprocessing.pool import ThreadPool
+import os
 import re
+import subprocess
+import sys
 
 
 def iwyu_formatter(output):
@@ -144,7 +146,7 @@ def get_output(cwd, args):
     return process.communicate()[0].decode("utf-8").splitlines()
 
 
-def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter):
+def run_iwyu(cwd, compile_command, iwyu_args, verbose):
     """ Rewrite compile_command to an IWYU command, and run it. """
     compiler, _, args = compile_command.partition(' ')
     if compiler.endswith('cl.exe'):
@@ -162,7 +164,10 @@ def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter):
     if verbose:
         print('{}'.format(cmd_args))
 
-    formatter(get_output(cwd, cmd_args))
+    return get_output(cwd, cmd_args)
+
+
+
 
 
 def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
@@ -203,14 +208,21 @@ def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
                       source)
 
     # Run analysis
+    def run_iwyu_task(entry):
+        cwd, compile_command = entry['directory'], entry['command']
+        return run_iwyu(cwd, compile_command, iwyu_args, verbose)
+    pool = ThreadPool(multiprocessing.cpu_count())
     try:
-        for entry in entries:
-            cwd, compile_command = entry['directory'], entry['command']
-            run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter)
+        for iwyu_output in pool.imap_unordered(run_iwyu_task, entries):
+            formatter(iwyu_output)
+    except KeyboardInterrupt as ki:
+        sys.exit(1)
     except OSError as why:
         print('ERROR: Failed to launch include-what-you-use: %s' % why)
         return 1
-
+    finally:
+        pool.terminate()
+        pool.join()
     return 0
 
 


Mime
View raw message