kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ale...@apache.org
Subject kudu git commit: [build-support] IWYU build configuration for Jenkins
Date Tue, 22 Aug 2017 14:29:43 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 9d12910a4 -> 8f6812047


[build-support] IWYU build configuration for Jenkins

Added provisions to run the include-what-you-use tool against the
updated files in the changelist.  Those are based on the newly
introduced 'iwyu' target to be used like already existing 'lint'
target.

The 'iwyu' target runs IWYU against the compilation database.  So,
as a part of this changelist I updated the top-level CMakeLists.txt
to always generate the compilation database.  The compilation database
is a JSON file containing information on how the targets are built.

Change-Id: I45f2e8529891dd7da18c9fa6383e0c41e6b4182e
Reviewed-on: http://gerrit.cloudera.org:8080/7750
Reviewed-by: Adar Dembo <adar@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8f681204
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8f681204
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8f681204

Branch: refs/heads/master
Commit: 8f6812047a2fe50dabfcb26093385b33f9960a7c
Parents: 9d12910
Author: Alexey Serbin <aserbin@cloudera.com>
Authored: Fri Aug 18 18:44:53 2017 -0700
Committer: Alexey Serbin <aserbin@cloudera.com>
Committed: Tue Aug 22 14:12:38 2017 +0000

----------------------------------------------------------------------
 .ycm_extra_conf.py                      |   4 -
 CMakeLists.txt                          |  19 +-
 build-support/iwyu/iwyu.sh              |  61 ++++++
 build-support/iwyu/iwyu_tool.py         | 272 +++++++++++++++++++++++++++
 build-support/jenkins/build-and-test.sh |  23 ++-
 5 files changed, 361 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/8f681204/.ycm_extra_conf.py
----------------------------------------------------------------------
diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py
index 2c1efbb..4e3d8ad 100644
--- a/.ycm_extra_conf.py
+++ b/.ycm_extra_conf.py
@@ -48,10 +48,6 @@ flags = get_flags()
 # compile_commands.json file to use that instead of 'flags'. See here for
 # more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html
 #
-# You can get CMake to generate this file for you by adding:
-#   set( CMAKE_EXPORT_COMPILE_COMMANDS 1 )
-# to your CMakeLists.txt file.
-#
 # Most projects will NOT need to set this to anything; you can just change the
 # 'flags' list of compilation flags. Notice that YCM itself uses that approach.
 compilation_database_folder = ''

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f681204/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 05a63ce..3ab6178 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -100,6 +100,11 @@ set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
 # just do it once here for all Kudu executables.
 set(CMAKE_ENABLE_EXPORTS true)
 
+# Always generate the compilation database file (compile_commands.json) for use
+# with various development tools, such as IWYU and Vim's YouCompleteMe plugin.
+# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
+set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
+
 # Make sure thirdparty stuff is up-to-date.
 if ("$ENV{NO_REBUILD_THIRDPARTY}" STREQUAL "")
   if (${KUDU_USE_TSAN})
@@ -113,13 +118,6 @@ if ("$ENV{NO_REBUILD_THIRDPARTY}" STREQUAL "")
   endif()
 endif()
 
-# Generate a Clang compile_commands.json "compilation database" file for use
-# with various development tools, such as Vim's YouCompleteMe plugin.
-# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
-if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
-  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
-endif()
-
 ############################################################
 # Compiler flags
 ############################################################
@@ -1052,6 +1050,13 @@ if (UNIX)
 endif (UNIX)
 
 ############################################################
+# "make iwyu" target
+############################################################
+if (UNIX)
+  add_custom_target(iwyu ${BUILD_SUPPORT_DIR}/iwyu/iwyu.sh)
+endif (UNIX)
+
+############################################################
 # "make docs" target
 ############################################################
 if (UNIX)

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f681204/build-support/iwyu/iwyu.sh
----------------------------------------------------------------------
diff --git a/build-support/iwyu/iwyu.sh b/build-support/iwyu/iwyu.sh
new file mode 100755
index 0000000..7cc087e
--- /dev/null
+++ b/build-support/iwyu/iwyu.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ROOT=$(cd $(dirname $BASH_SOURCE)/../..; pwd)
+
+IWYU_LOG=$(mktemp -t kudu-iwyu.XXXXXX)
+trap "rm -f $IWYU_LOG" EXIT
+
+# Build the list of updated files which are of IWYU interest.
+file_list_tmp=$(git diff --name-only \
+    $($ROOT/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
+if [ -z "$file_list_tmp" ]; then
+  echo "IWYU verification: no updates on related files, declaring success"
+  exit 0
+fi
+
+# Adjust the path for every element in the list. The iwyu_tool.py normalizes
+# paths (via realpath) to match the records from the compilation database.
+IWYU_FILE_LIST=
+for p in $file_list_tmp; do
+  IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
+done
+
+IWYU_MAPPINGS_PATH="$ROOT/build-support/iwyu/mappings"
+IWYU_ARGS="\
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-all-private.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
+    --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp"
+
+PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \
+    python $ROOT/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \
+    $IWYU_ARGS | awk -f $ROOT/build-support/iwyu/iwyu-filter.awk | \
+    tee $IWYU_LOG
+
+if [ -s "$IWYU_LOG" ]; then
+  # The output is not empty: the changelist needs correction.
+  exit 1
+fi
+
+# The output is empty: the changelist looks good.
+echo "IWYU verification: the changes look good"
+exit 0

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f681204/build-support/iwyu/iwyu_tool.py
----------------------------------------------------------------------
diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py
new file mode 100755
index 0000000..a3dd4a1
--- /dev/null
+++ b/build-support/iwyu/iwyu_tool.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+
+# This file has been imported into the Kudu source tree from
+# the IWYU source tree as of version 0.8
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/iwyu_tool.py
+# and corresponding license has been added:
+#   https://github.com/include-what-you-use/include-what-you-use/blob/master/LICENSE.TXT
+#
+# ==============================================================================
+# LLVM Release License
+# ==============================================================================
+# University of Illinois/NCSA
+# Open Source License
+#
+# Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign.
+# All rights reserved.
+#
+# Developed by:
+#
+#     LLVM Team
+#
+#     University of Illinois at Urbana-Champaign
+#
+#     http://llvm.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal with
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimers.
+#
+#     * Redistributions in binary form must reproduce the above copyright notice,
+#       this list of conditions and the following disclaimers in the
+#       documentation and/or other materials provided with the distribution.
+#
+#     * Neither the names of the LLVM Team, University of Illinois at
+#       Urbana-Champaign, nor the names of its contributors may be used to
+#       endorse or promote products derived from this Software without specific
+#       prior written permission.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+# SOFTWARE.
+
+""" Driver to consume a Clang compilation database and invoke IWYU.
+
+Example usage with CMake:
+
+  # Unix systems
+  $ mkdir build && cd build
+  $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
+  $ iwyu_tool.py -p .
+
+  # Windows systems
+  $ mkdir build && cd build
+  $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
+    -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+    -G Ninja ...
+  $ python iwyu_tool.py -p .
+
+See iwyu_tool.py -h for more details on command-line arguments.
+"""
+
+import os
+import sys
+import json
+import argparse
+import subprocess
+import re
+
+
+def iwyu_formatter(output):
+    """ Process iwyu's output, basically a no-op. """
+    print('\n'.join(output))
+
+
+CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
+SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
+SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
+FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
+END_RE = re.compile(r'^---$')
+LINES_RE = re.compile(r'^- (.*?)  // lines ([0-9]+)-[0-9]+$')
+
+
+GENERAL, ADD, REMOVE, LIST = range(4)
+
+
+def clang_formatter(output):
+    """ Process iwyu's output into something clang-like. """
+    state = (GENERAL, None)
+    for line in output:
+        match = CORRECT_RE.match(line)
+        if match:
+            print('%s:1:1: note: #includes/fwd-decls are correct' % match.groups(1))
+            continue
+        match = SHOULD_ADD_RE.match(line)
+        if match:
+            state = (ADD, match.group(1))
+            continue
+        match = SHOULD_REMOVE_RE.match(line)
+        if match:
+            state = (REMOVE, match.group(1))
+            continue
+        match = FULL_LIST_RE.match(line)
+        if match:
+            state = (LIST, match.group(1))
+        elif END_RE.match(line):
+            state = (GENERAL, None)
+        elif not line.strip():
+            continue
+        elif state[0] == GENERAL:
+            print(line)
+        elif state[0] == ADD:
+            print('%s:1:1: error: add the following line' % state[1])
+            print(line)
+        elif state[0] == REMOVE:
+            match = LINES_RE.match(line)
+            line_no = match.group(2) if match else '1'
+            print('%s:%s:1: error: remove the following line' % (state[1], line_no))
+            print(match.group(1))
+
+
+DEFAULT_FORMAT = 'iwyu'
+FORMATTERS = {
+    'iwyu': iwyu_formatter,
+    'clang': clang_formatter
+}
+
+def get_output(cwd, command):
+    """ Run the given command and return its output as a string. """
+    process = subprocess.Popen(command,
+                               cwd=cwd,
+                               shell=True,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT)
+    return process.communicate()[0].decode("utf-8").splitlines()
+
+
+def run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter):
+    """ Rewrite compile_command to an IWYU command, and run it. """
+    compiler, _, args = compile_command.partition(' ')
+    if compiler.endswith('cl.exe'):
+        # If the compiler name is cl.exe, let IWYU be cl-compatible
+        clang_args = ['--driver-mode=cl']
+    else:
+        clang_args = []
+
+    iwyu_args = ['-Xiwyu ' + a for a in iwyu_args]
+    command = ['include-what-you-use'] + clang_args + iwyu_args
+    command = '%s %s' % (' '.join(command), args.strip())
+
+    if verbose:
+        print('%s:' % command)
+
+    formatter(get_output(cwd, command))
+
+
+def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
+    """ Entry point. """
+    # Canonicalize compilation database path
+    if os.path.isdir(compilation_db_path):
+        compilation_db_path = os.path.join(compilation_db_path,
+                                           'compile_commands.json')
+
+    compilation_db_path = os.path.realpath(compilation_db_path)
+    if not os.path.isfile(compilation_db_path):
+        print('ERROR: No such file or directory: \'%s\'' % compilation_db_path)
+        return 1
+
+    # Read compilation db from disk
+    with open(compilation_db_path, 'r') as fileobj:
+        compilation_db = json.load(fileobj)
+
+    # expand symlinks
+    for entry in compilation_db:
+        entry['file'] = os.path.realpath(entry['file'])
+
+    # Cross-reference source files with compilation database
+    source_files = [os.path.realpath(s) for s in source_files]
+    if not source_files:
+        # No source files specified, analyze entire compilation database
+        entries = compilation_db
+    else:
+        # Source files specified, analyze the ones appearing in compilation db,
+        # warn for the rest.
+        entries = []
+        for source in source_files:
+            matches = [e for e in compilation_db if e['file'] == source]
+            if matches:
+                entries.extend(matches)
+            else:
+                print('WARNING: \'%s\' not found in compilation database.' %
+                      source)
+
+    # Run analysis
+    try:
+        for entry in entries:
+            cwd, compile_command = entry['directory'], entry['command']
+            run_iwyu(cwd, compile_command, iwyu_args, verbose, formatter)
+    except OSError as why:
+        print('ERROR: Failed to launch include-what-you-use: %s' % why)
+        return 1
+
+    return 0
+
+
+def _bootstrap():
+    """ Parse arguments and dispatch to main(). """
+    # This hackery is necessary to add the forwarded IWYU args to the
+    # usage and help strings.
+    def customize_usage(parser):
+        """ Rewrite the parser's format_usage. """
+        original_format_usage = parser.format_usage
+        parser.format_usage = lambda: original_format_usage().rstrip() + \
+                              ' -- [<IWYU args>]' + os.linesep
+
+    def customize_help(parser):
+        """ Rewrite the parser's format_help. """
+        original_format_help = parser.format_help
+
+        def custom_help():
+            """ Customized help string, calls the adjusted format_usage. """
+            helpmsg = original_format_help()
+            helplines = helpmsg.splitlines()
+            helplines[0] = parser.format_usage().rstrip()
+            return os.linesep.join(helplines) + os.linesep
+
+        parser.format_help = custom_help
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Include-what-you-use compilation database driver.',
+        epilog='Assumes include-what-you-use is available on the PATH.')
+    customize_usage(parser)
+    customize_help(parser)
+
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Print IWYU commands')
+    parser.add_argument('-o', '--output-format', type=str,
+                        choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
+                        help='Output format (default: %s)' % DEFAULT_FORMAT)
+    parser.add_argument('-p', metavar='<build-path>', required=True,
+                        help='Compilation database path', dest='dbpath')
+    parser.add_argument('source', nargs='*',
+                        help='Zero or more source files to run IWYU on. '
+                        'Defaults to all in compilation database.')
+
+    def partition_args(argv):
+        """ Split around '--' into driver args and IWYU args. """
+        try:
+            double_dash = argv.index('--')
+            return argv[:double_dash], argv[double_dash+1:]
+        except ValueError:
+            return argv, []
+    argv, iwyu_args = partition_args(sys.argv[1:])
+    args = parser.parse_args(argv)
+
+    sys.exit(main(args.dbpath, args.source, args.verbose,
+                  FORMATTERS[args.output_format], iwyu_args))
+
+
+if __name__ == '__main__':
+    _bootstrap()

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f681204/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index 6b735a3..0a6f567 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -22,7 +22,7 @@
 #
 # Environment variables may be used to customize operation:
 #   BUILD_TYPE: Default: DEBUG
-#     Maybe be one of ASAN|TSAN|DEBUG|RELEASE|COVERAGE|LINT
+#     Maybe be one of ASAN|TSAN|DEBUG|RELEASE|COVERAGE|LINT|IWYU
 #
 #   KUDU_ALLOW_SLOW_TESTS   Default: 1
 #     Runs the "slow" version of the unit tests. Set to 0 to
@@ -196,6 +196,9 @@ elif [ "$BUILD_TYPE" = "COVERAGE" ]; then
   BUILD_PYTHON3=0
 elif [ "$BUILD_TYPE" = "LINT" ]; then
   CMAKE_BUILD=debug
+elif [ "$BUILD_TYPE" = "IWYU" ]; then
+  USE_CLANG=1
+  CMAKE_BUILD=debug
 else
   # Must be DEBUG or RELEASE
   CMAKE_BUILD=$BUILD_TYPE
@@ -222,17 +225,24 @@ fi
 CMAKE="$CMAKE $SOURCE_ROOT"
 $CMAKE
 
+# Create empty test logs or else Jenkins fails to archive artifacts, which
+# results in the build failing.
+mkdir -p Testing/Temporary
+mkdir -p $TEST_LOGDIR
+
 # Short circuit for LINT builds.
 if [ "$BUILD_TYPE" = "LINT" ]; then
-  # Create empty test logs or else Jenkins fails to archive artifacts, which
-  # results in the build failing.
-  mkdir -p Testing/Temporary
-  mkdir -p $TEST_LOGDIR
-
   make lint | tee $TEST_LOGDIR/lint.log
   exit $?
 fi
 
+# Short circuit for IWYU builds: run the include-what-you-use tool on the files
+# modified since the last committed changelist committed upstream.
+if [ "$BUILD_TYPE" = "IWYU" ]; then
+  make iwyu | tee $TEST_LOGDIR/iwyu.log
+  exit $?
+fi
+
 # Only enable test core dumps for certain build types.
 if [ "$BUILD_TYPE" != "ASAN" ]; then
   export KUDU_TEST_ULIMIT_CORE=unlimited
@@ -273,7 +283,6 @@ set +e
 
 # Run tests
 export GTEST_OUTPUT="xml:$TEST_LOGDIR/" # Enable JUnit-compatible XML output.
-mkdir -p $TEST_LOGDIR # In case no tests are run (the directory won't be created)
 if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
   if [ -z "$TEST_RESULT_SERVER" ]; then
     echo Must set TEST_RESULT_SERVER to use RUN_FLAKY_ONLY


Mime
View raw message