yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [yetus] branch master updated: YETUS-788. expose patch analysis
Date Tue, 05 Feb 2019 05:42:52 GMT
This is an automated email from the ASF dual-hosted git repository.

aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git


The following commit(s) were added to refs/heads/master by this push:
     new 3db66f6  YETUS-788. expose patch analysis
3db66f6 is described below

commit 3db66f617bb87df6bcc378bb39a9de0af6c67190
Author: Allen Wittenauer <aw@apache.org>
AuthorDate: Fri Feb 1 14:00:21 2019 -0800

    YETUS-788. expose patch analysis
    
    Signed-off-by: Allen Wittenauer <aw@apache.org>
---
 precommit/src/main/shell/core.d/01-common.sh       |  16 ++
 precommit/src/main/shell/core.d/change-analysis.sh | 236 +++++++++++++++++++++
 precommit/src/main/shell/smart-apply-patch.sh      |  89 +++++++-
 precommit/src/main/shell/test-patch.sh             | 225 --------------------
 4 files changed, 340 insertions(+), 226 deletions(-)

diff --git a/precommit/src/main/shell/core.d/01-common.sh b/precommit/src/main/shell/core.d/01-common.sh
index 6c58021..0692a9c 100755
--- a/precommit/src/main/shell/core.d/01-common.sh
+++ b/precommit/src/main/shell/core.d/01-common.sh
@@ -712,4 +712,20 @@ function guess_build_tool
   fi
 
   echo "Setting build tool to ${BUILDTOOL}"
+}
+
+## @description  Convert the given module name to a file fragment
+## @audience     public
+## @stability    stable
+## @replaceable  no
+## @param        module
+function module_file_fragment
+{
+  local mod=$1
+  if [[ ${mod} = \. ]]; then
+    echo root
+  else
+    #shellcheck disable=SC1003
+    echo "$1" | tr '/' '_' | tr '\\' '_'
+  fi
 }
\ No newline at end of file
diff --git a/precommit/src/main/shell/core.d/change-analysis.sh b/precommit/src/main/shell/core.d/change-analysis.sh
new file mode 100755
index 0000000..4b1fb13
--- /dev/null
+++ b/precommit/src/main/shell/core.d/change-analysis.sh
@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+## @description  Locate the build file for a given directory
+## @audience     private
+## @stability    stable
+## @replaceable  no
+## @return       directory containing the buildfile. Nothing returned if not found.
+## @param        buildfile
+## @param        directory
+function find_buildfile_dir
+{
+  local buildfile=$1
+  local dir=$2
+
+  yetus_debug "Find ${buildfile} dir for: ${dir}"
+
+  while builtin true; do
+    if [[ -f "${dir}/${buildfile}" ]];then
+      echo "${dir}"
+      yetus_debug "Found: ${dir}"
+      return 0
+    elif [[ ${dir} == "." || ${dir} == "/" ]]; then
+      yetus_debug "ERROR: ${buildfile} is not found."
+      return 1
+    else
+      dir=$(faster_dirname "${dir}")
+    fi
+  done
+}
+
+## @description  List of files that ${PATCH_DIR}/patch modifies
+## @audience     private
+## @stability    stable
+## @replaceable  no
+## @return       None; sets ${CHANGED_FILES[@]}
+function find_changed_files
+{
+  declare line
+
+  BUILDMODE=${BUILDMODE:-patch}
+
+  pushd "${BASEDIR}" >/dev/null || return 1
+
+  case "${BUILDMODE}" in
+    full)
+      echo "Building a list of all files in the source tree"
+      while IFS= read -r; do CHANGED_FILES+=("$REPLY"); done < <("${GIT}" ls-files)
+    ;;
+    patch)
+      # get a list of all of the files that have been changed,
+      # except for /dev/null (which would be present for new files).
+      # Additionally, remove any a/ b/ patterns at the front of the patch filenames.
+      # shellcheck disable=SC2016
+      while read -r line; do
+        CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
+      done < <(
+        "${AWK}" 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}}
+        /^diff --git /   { p($3); p($4) }
+        /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
+      ;;
+    esac
+  popd >/dev/null || return 1
+}
+
+## @description Check for directories to skip during
+## @description changed module calcuation
+## @description requires $MODULE_SKIPDIRS to be set
+## @audience    private
+## @stability   stable
+## @replaceable no
+## @param       directory
+## @return      0 for use
+## @return      1 for skip
+function module_skipdir
+{
+  local dir=${1}
+  local i
+
+  yetus_debug "Checking skipdirs for ${dir}"
+
+  if [[ -z ${MODULE_SKIPDIRS} ]]; then
+    yetus_debug "Skipping skipdirs"
+    return 0
+  fi
+
+  while builtin true; do
+    for i in ${MODULE_SKIPDIRS}; do
+      if [[ ${dir} = "${i}" ]];then
+        yetus_debug "Found a skip: ${dir}"
+        return 1
+      fi
+    done
+    if [[ ${dir} == "." || ${dir} == "/" ]]; then
+      return 0
+    else
+      dir=$(faster_dirname "${dir}")
+      yetus_debug "Trying to skip: ${dir}"
+    fi
+  done
+}
+
+## @description  Find the modules of the build that ${PATCH_DIR}/patch modifies
+## @audience     private
+## @stability    stable
+## @replaceable  no
+## @param        repostatus
+## @return       None; sets ${CHANGED_MODULES[@]}
+function find_changed_modules
+{
+  declare repostatus=$1
+  declare i
+  declare builddir
+  declare module
+  declare prev_builddir
+  declare i=1
+  declare dir
+  declare dirt
+  declare buildfile
+  declare -a tmpmods
+  declare retval
+
+  if declare -f "${BUILDTOOL}_buildfile" >/dev/null; then
+    buildfile=$("${BUILDTOOL}_buildfile")
+    retval=$?
+  else
+    yetus_error "ERROR: build tool plugin is broken"
+    bugsystem_finalreport 1
+    cleanup_and_exit 1
+  fi
+
+  if [[ ${retval} != 0 ]]; then
+    yetus_error "ERROR: Unsupported build tool."
+    bugsystem_finalreport 1
+    cleanup_and_exit 1
+  fi
+
+  pushd "${BASEDIR}" >/dev/null || return 1
+
+  #  Empty string indicates the build system wants to disable module detection
+  if [[ -z ${buildfile} ]]; then
+    CHANGED_MODULES=(".")
+  else
+
+    # Now find all the modules that were changed
+    for i in "${CHANGED_FILES[@]}"; do
+
+      # TODO: optimize this
+      if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then
+        continue
+      fi
+
+      dirt=$(dirname "${i}")
+
+      if ! module_skipdir "${dirt}"; then
+        continue
+      fi
+
+      builddir=$(find_buildfile_dir "${buildfile}" "${dirt}")
+      if [[ -z ${builddir} ]]; then
+        yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based
project."
+        bugsystem_finalreport 1
+        cleanup_and_exit 1
+      fi
+      CHANGED_MODULES+=("${builddir}")
+    done
+  fi
+
+  CHANGED_MODULES+=("${USER_MODULE_LIST[@]}")
+
+  yetus_sort_and_unique_array CHANGED_MODULES
+
+  yetus_debug "Locate the union of ${CHANGED_MODULES[*]}"
+
+  count=${#CHANGED_MODULES[@]}
+  if [[ ${count} -lt 2 ]]; then
+    yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES[0]}"
+    # shellcheck disable=SC2034
+    CHANGED_UNION_MODULES="${CHANGED_MODULES[0]}"
+  else
+    i=1
+
+    # BUG - fix me
+    # shellcheck disable=SC2207
+    while [[ ${i} -lt 100 ]]; do
+      tmpmods=()
+      for j in "${CHANGED_MODULES[@]}"; do
+        tmpmods+=($(echo "${j}" | cut -f1-${i} -d/))
+      done
+      tmpmods=($(printf '%s\n' "${tmpmods[@]}" | sort -u))
+
+      module=${tmpmods[0]}
+      count=${#tmpmods[@]}
+      if [[ ${count} -eq 1
+        && -f ${module}/${buildfile} ]]; then
+        prev_builddir=${module}
+      elif [[ ${count} -gt 1 ]]; then
+        builddir=${prev_builddir}
+        break
+      fi
+      ((i=i+1))
+    done
+
+    if [[ -z ${builddir} ]]; then
+      builddir="."
+    fi
+
+    yetus_debug "Finding union of ${builddir}"
+    builddir=$(find_buildfile_dir "${buildfile}" "${builddir}" || true)
+
+    #shellcheck disable=SC2034
+    CHANGED_UNION_MODULES="${builddir}"
+  fi
+
+  # some build tools may want to change these and/or
+  # make other changes based upon these results
+  if declare -f "${BUILDTOOL}_reorder_modules" >/dev/null; then
+    "${BUILDTOOL}_reorder_modules" "${repostatus}"
+  fi
+  popd >/dev/null || return 1
+
+}
diff --git a/precommit/src/main/shell/smart-apply-patch.sh b/precommit/src/main/shell/smart-apply-patch.sh
index bdee4cb..0a5a742 100755
--- a/precommit/src/main/shell/smart-apply-patch.sh
+++ b/precommit/src/main/shell/smart-apply-patch.sh
@@ -27,6 +27,8 @@ this="${BASH_SOURCE-$0}"
 BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
 #shellcheck disable=SC2034
 QATESTMODE=false
+STARTINGDIR=$(pwd)
+
 
 ## @description  disable function
 ## @stability    stable
@@ -64,6 +66,16 @@ function add_test
   true
 }
 
+
+## @description  disable function
+## @stability    stable
+## @audience     private
+## @replaceable  no
+function bugsystem_finalreport
+{
+  true
+}
+
 ## @description  Clean the filesystem as appropriate and then exit
 ## @audience     private
 ## @stability    evolving
@@ -135,6 +147,15 @@ function yetus_usage
   yetus_reset_usage
 
   echo ""
+  echo "Patch reporting:"
+  yetus_add_option "--build-tool=<tool>" "Override the build tool"
+  yetus_add_option "--changedfilesreport=<name>" "List of files that this patch modifies"
+  yetus_add_option "--changedmodulesreport=<name>" "List of modules that this patch
modifies"
+  yetus_add_option "--changedunionreport=<name>" "Union of modules that this patch
modifies"
+  yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+  yetus_reset_usage
+
+  echo ""
   importplugins
 
   unset TESTFORMATS
@@ -166,6 +187,9 @@ function parse_args
 
   for i in "$@"; do
     case ${i} in
+      --build-tool=*)
+        BUILDTOOL=${i#*=}
+      ;;
       --committer)
         COMMITMODE=true
       ;;
@@ -175,6 +199,15 @@ function parse_args
       --dry-run)
         PATCH_DRYRUNMODE=true
       ;;
+      --changedfilesreport=*)
+        FILEREPORT=${i#*=}
+      ;;
+      --changedmodulesreport=*)
+        MODULEREPORT=${i#*=}
+      ;;
+      --changedunionreport=*)
+        UNIONREPORT=${i#*=}
+      ;;
       --*)
         ## PATCH_OR_ISSUE can't be a --.  So this is probably
         ## a plugin thing.
@@ -192,6 +225,55 @@ function parse_args
       cleanup_and_exit 1
     fi
   fi
+
+  # we need absolute dir for ${BASEDIR}
+  cd "${STARTINGDIR}" || cleanup_and_exit 1
+  BASEDIR=$(yetus_abs "${BASEDIR}")
+}
+
+## @description  generate reports on the given patch file
+## @replaceable  no
+## @audience     private
+## @stability    evolving
+function patch_reports
+{
+  declare i
+
+  if [[ -z "${FILEREPORT}" ]] && [[ -z "${MODULEREPORT}" ]] && [[ -z "${UNIONREPORT}"
]]; then
+    return
+  fi
+
+  set -x
+
+  find_changed_files
+
+  if [[ -n "${FILEREPORT}" ]]; then
+    : > "${FILEREPORT}"
+    for i in "${CHANGED_FILES[@]}"; do
+      echo "${i}" >> "${FILEREPORT}"
+    done
+  fi
+
+  if [[ -n "${MODULEREPORT}" ]] || [[ -n "${UNIONREPORT}" ]]; then
+    if [[ -z "${BUILDTOOL}" ]]; then
+      guess_build_tool
+    fi
+
+    unset -f "${BUILDTOOL}_reorder_modules"
+
+    find_changed_modules
+
+    if [[ -n "${MODULEREPORT}" ]]; then
+      : > "${MODULEREPORT}"
+      for i in "${CHANGED_MODULES[@]}"; do
+        echo "${i}" >> "${MODULEREPORT}"
+      done
+    fi
+
+    if [[ -n "${UNIONREPORT}" ]]; then
+      cat "${CHANGED_UNION_MODULES}" > "${UNIONREPORT}"
+    fi
+  fi
 }
 
 ## @description  git am dryrun
@@ -305,7 +387,6 @@ parse_args "$@"
 
 importplugins
 yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list"
-unset BUILDTOOLS
 unset TESTTYPES
 unset TESTFORMATS
 
@@ -333,6 +414,8 @@ if [[ ${RESULT} -gt 0 ]]; then
   cleanup_and_exit ${RESULT}
 fi
 
+pushd "${BASEDIR}" >/dev/null || exit 1
+
 if [[ ${PATCH_DRYRUNMODE} == false ]]; then
   patchfile_apply_driver "${PATCH_DIR}/patch" "${GPGSIGN}"
   RESULT=$?
@@ -344,4 +427,8 @@ if [[ ${COMMITMODE} = true
   git add -A
 fi
 
+patch_reports
+
+popd >/dev/null || exit 1
+
 cleanup_and_exit ${RESULT}
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index dd0bae5..b84fb57 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -103,21 +103,6 @@ function setup_defaults
   yetus_add_entry JDK_TEST_LIST unit
 }
 
-## @description  Convert the given module name to a file fragment
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        module
-function module_file_fragment
-{
-  local mod=$1
-  if [[ ${mod} = \. ]]; then
-    echo root
-  else
-    #shellcheck disable=SC1003
-    echo "$1" | tr '/' '_' | tr '\\' '_'
-  fi
-}
 
 ## @description  Convert time in seconds to m + s
 ## @audience     public
@@ -1080,216 +1065,6 @@ function set_buildmode
   fi
 }
 
-## @description  Locate the build file for a given directory
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       directory containing the buildfile. Nothing returned if not found.
-## @param        buildfile
-## @param        directory
-function find_buildfile_dir
-{
-  local buildfile=$1
-  local dir=$2
-
-  yetus_debug "Find ${buildfile} dir for: ${dir}"
-
-  while builtin true; do
-    if [[ -f "${dir}/${buildfile}" ]];then
-      echo "${dir}"
-      yetus_debug "Found: ${dir}"
-      return 0
-    elif [[ ${dir} == "." || ${dir} == "/" ]]; then
-      yetus_debug "ERROR: ${buildfile} is not found."
-      return 1
-    else
-      dir=$(faster_dirname "${dir}")
-    fi
-  done
-}
-
-## @description  List of files that ${PATCH_DIR}/patch modifies
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       None; sets ${CHANGED_FILES[@]}
-function find_changed_files
-{
-  declare line
-
-  case "${BUILDMODE}" in
-    full)
-      echo "Building a list of all files in the source tree"
-      while IFS= read -r; do CHANGED_FILES+=("$REPLY"); done < <(git ls-files)
-    ;;
-    patch)
-      # get a list of all of the files that have been changed,
-      # except for /dev/null (which would be present for new files).
-      # Additionally, remove any a/ b/ patterns at the front of the patch filenames.
-      # shellcheck disable=SC2016
-      while read -r line; do
-        CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
-      done < <(
-        ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}}
-        /^diff --git /   { p($3); p($4) }
-        /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
-      ;;
-    esac
-}
-
-## @description Check for directories to skip during
-## @description changed module calcuation
-## @audience    private
-## @stability   stable
-## @replaceable no
-## @param       directory
-## @return      0 for use
-## @return      1 for skip
-function module_skipdir
-{
-  local dir=${1}
-  local i
-
-  yetus_debug "Checking skipdirs for ${dir}"
-
-  if [[ -z ${MODULE_SKIPDIRS} ]]; then
-    yetus_debug "Skipping skipdirs"
-    return 0
-  fi
-
-  while builtin true; do
-    for i in ${MODULE_SKIPDIRS}; do
-      if [[ ${dir} = "${i}" ]];then
-        yetus_debug "Found a skip: ${dir}"
-        return 1
-      fi
-    done
-    if [[ ${dir} == "." || ${dir} == "/" ]]; then
-      return 0
-    else
-      dir=$(faster_dirname "${dir}")
-      yetus_debug "Trying to skip: ${dir}"
-    fi
-  done
-}
-
-## @description  Find the modules of the build that ${PATCH_DIR}/patch modifies
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @param        repostatus
-## @return       None; sets ${CHANGED_MODULES[@]}
-function find_changed_modules
-{
-  declare repostatus=$1
-  declare i
-  declare builddir
-  declare module
-  declare prev_builddir
-  declare i=1
-  declare dir
-  declare dirt
-  declare buildfile
-  declare -a tmpmods
-  declare retval
-
-  if declare -f "${BUILDTOOL}_buildfile" >/dev/null; then
-    buildfile=$("${BUILDTOOL}_buildfile")
-    retval=$?
-  else
-    yetus_error "ERROR: build tool plugin is broken"
-    bugsystem_finalreport 1
-    cleanup_and_exit 1
-  fi
-
-  if [[ ${retval} != 0 ]]; then
-    yetus_error "ERROR: Unsupported build tool."
-    bugsystem_finalreport 1
-    cleanup_and_exit 1
-  fi
-
-  #  Empty string indicates the build system wants to disable module detection
-  if [[ -z ${buildfile} ]]; then
-    CHANGED_MODULES=(".")
-  else
-
-    # Now find all the modules that were changed
-    for i in "${CHANGED_FILES[@]}"; do
-
-      # TODO: optimize this
-      if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then
-        continue
-      fi
-
-      dirt=$(dirname "${i}")
-
-      if ! module_skipdir "${dirt}"; then
-        continue
-      fi
-
-      builddir=$(find_buildfile_dir "${buildfile}" "${dirt}")
-      if [[ -z ${builddir} ]]; then
-        yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based
project."
-        bugsystem_finalreport 1
-        cleanup_and_exit 1
-      fi
-      CHANGED_MODULES+=("${builddir}")
-    done
-  fi
-
-  CHANGED_MODULES+=("${USER_MODULE_LIST[@]}")
-
-  yetus_sort_and_unique_array CHANGED_MODULES
-
-  yetus_debug "Locate the union of ${CHANGED_MODULES[*]}"
-
-  count=${#CHANGED_MODULES[@]}
-  if [[ ${count} -lt 2 ]]; then
-    yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES[0]}"
-    # shellcheck disable=SC2034
-    CHANGED_UNION_MODULES="${CHANGED_MODULES[0]}"
-  else
-    i=1
-
-    # BUG - fix me
-    # shellcheck disable=SC2207
-    while [[ ${i} -lt 100 ]]; do
-      tmpmods=()
-      for j in "${CHANGED_MODULES[@]}"; do
-        tmpmods+=($(echo "${j}" | cut -f1-${i} -d/))
-      done
-      tmpmods=($(printf '%s\n' "${tmpmods[@]}" | sort -u))
-
-      module=${tmpmods[0]}
-      count=${#tmpmods[@]}
-      if [[ ${count} -eq 1
-        && -f ${module}/${buildfile} ]]; then
-        prev_builddir=${module}
-      elif [[ ${count} -gt 1 ]]; then
-        builddir=${prev_builddir}
-        break
-      fi
-      ((i=i+1))
-    done
-
-    if [[ -z ${builddir} ]]; then
-      builddir="."
-    fi
-
-    yetus_debug "Finding union of ${builddir}"
-    builddir=$(find_buildfile_dir "${buildfile}" "${builddir}" || true)
-
-    #shellcheck disable=SC2034
-    CHANGED_UNION_MODULES="${builddir}"
-  fi
-
-  # some build tools may want to change these and/or
-  # make other changes based upon these results
-  if declare -f "${BUILDTOOL}_reorder_modules" >/dev/null; then
-    "${BUILDTOOL}_reorder_modules" "${repostatus}"
-  fi
-}
-
 ## @description  check if repo requires creds to do remote operations
 ## @description  also sets and uses GIT_OFFLINE as appropriate
 ## @audience     private


Mime
View raw message