kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [5/5] kudu git commit: tidy: run in parallel
Date Wed, 18 Apr 2018 23:52:07 GMT
tidy: run in parallel

This makes the 'tidy' target run in parallel across all cores on the
machine, speeding it up substantially.

Change-Id: Idc611a32f55fa830d1ce7bed6c7cff6f6291d508
Reviewed-on: http://gerrit.cloudera.org:8080/10107
Reviewed-by: Dan Burkert <danburkert@apache.org>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a9271b05
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a9271b05
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a9271b05

Branch: refs/heads/master
Commit: a9271b05d1de4d922ce9c1bc40dc2baa5e4c7f7a
Parents: b87b026
Author: Todd Lipcon <todd@apache.org>
Authored: Wed Apr 18 14:30:07 2018 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Wed Apr 18 23:51:38 2018 +0000

----------------------------------------------------------------------
 build-support/clang_tidy_gerrit.py | 49 +++++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/a9271b05/build-support/clang_tidy_gerrit.py
----------------------------------------------------------------------
diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py
index b139d82..3e77abe 100755
--- a/build-support/clang_tidy_gerrit.py
+++ b/build-support/clang_tidy_gerrit.py
@@ -22,6 +22,8 @@ import collections
 import compile_flags
 import json
 import logging
+import multiprocessing
+from multiprocessing.pool import ThreadPool
 import os
 import re
 import subprocess
@@ -42,20 +44,39 @@ GERRIT_PASSWORD = os.getenv('TIDYBOT_PASSWORD')
 GERRIT_URL = "https://gerrit.cloudera.org"
 
 def run_tidy(sha="HEAD", is_rev_range=False):
-    patch_file = tempfile.NamedTemporaryFile()
-    cmd = [
-        "git", "diff" if is_rev_range else "show", sha,
-        "--src-prefix=%s/" % ROOT,
-        "--dst-prefix=%s/" % ROOT]
-    subprocess.check_call(cmd,
-                          stdout=patch_file)
-    return subprocess.check_output(
-        [CLANG_TIDY_DIFF,
-         "-clang-tidy-binary", CLANG_TIDY,
-         "-p0",
-         "--",
-         "-DCLANG_TIDY"] + compile_flags.get_flags(),
-        stdin=file(patch_file.name))
+    diff_cmdline = ["git", "diff" if is_rev_range else "show", sha]
+
+    # Figure out which paths changed in the given diff.
+    changed_paths = subprocess.check_output(diff_cmdline + ["--name-only", "--pretty=format:"]).splitlines()
+    changed_paths = [p for p in changed_paths if p]
+
+    # Produce a separate diff for each file and run clang-tidy-diff on it
+    # in parallel.
+    def tidy_on_path(path):
+        patch_file = tempfile.NamedTemporaryFile()
+        cmd = diff_cmdline + [
+            "--src-prefix=%s/" % ROOT,
+            "--dst-prefix=%s/" % ROOT,
+            "--",
+            path]
+        subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT)
+        cmdline = [CLANG_TIDY_DIFF,
+                   "-clang-tidy-binary", CLANG_TIDY,
+                   "-p0",
+                   "--",
+                   "-DCLANG_TIDY"] + compile_flags.get_flags()
+        return subprocess.check_output(
+            cmdline,
+            stdin=file(patch_file.name),
+            cwd=ROOT)
+    pool = ThreadPool(multiprocessing.cpu_count())
+    try:
+        return "".join(pool.imap(tidy_on_path, changed_paths))
+    except KeyboardInterrupt as ki:
+        sys.exit(1)
+    finally:
+        pool.terminate()
+        pool.join()
 
 
 def split_warnings(clang_output):


Mime
View raw message