hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [21/21] hadoop git commit: HADOOP-12226. CHANGED_MODULES is wrong for ant (aw)
Date Thu, 16 Jul 2015 17:37:06 GMT
HADOOP-12226. CHANGED_MODULES is wrong for ant (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/840e0e5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/840e0e5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/840e0e5f

Branch: refs/heads/HADOOP-12111
Commit: 840e0e5f7808790e53cbb67bccf7e216b2f84034
Parents: b67f74b
Author: Allen Wittenauer <aw@apache.org>
Authored: Thu Jul 16 10:36:36 2015 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Thu Jul 16 10:36:36 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.sh | 224 +++++++++++++++++++++++++++--------------
 1 file changed, 151 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/840e0e5f/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 327e6f1..24dafc3 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -84,8 +84,12 @@ function setup_defaults
   PATCH_BRANCH=""
   PATCH_BRANCH_DEFAULT="master"
 
-  #shellcheck disable=SC2034
+  # shellcheck disable=SC2034
   CHANGED_MODULES=""
+  # shellcheck disable=SC2034
+  CHANGED_UNFILTERED_MODULES=""
+  # shellcheck disable=SC2034
+  CHANGED_UNION_MODULES=""
   USER_MODULE_LIST=""
   OFFLINE=false
   CHANGED_FILES=""
@@ -656,10 +660,12 @@ function echo_and_redirect
 
   find "${BASEDIR}" -type d -exec chmod +x {} \;
   # to the screen
+  echo "cd $(pwd)"
   echo "${*} > ${logfile} 2>&1"
   # to the log
-  echo "${*}" > "${logfile}"
-  # the actual command
+  echo "cd $(pwd)" > "${logfile}"
+  echo "${*}" >> "${logfile}"
+  # run the actual command
   "${@}" >> "${logfile}" 2>&1
 }
 
@@ -723,6 +729,7 @@ function testpatch_usage
   echo "--project=<name>       The short name for project currently using test-patch
(default 'yetus')"
   echo "--resetrepo            Forcibly clean the repo"
   echo "--run-tests            Run all relevant tests below the base directory"
+  echo "--skip-dirs=<list>     Skip following directories for module finding"
   echo "--skip-system-plugins  Do not load plugins from ${BINDIR}/test-patch.d"
   echo "--summarize=<bool>     Allow tests to summarize results"
   echo "--testlist=<list>      Specify which subsystem tests to use (comma delimited)"
@@ -768,6 +775,7 @@ function parse_args
 {
   local i
   local j
+  local testlist
 
   for i in "$@"; do
     case ${i} in
@@ -893,6 +901,11 @@ function parse_args
       --run-tests)
         RUN_TESTS=true
       ;;
+      --skip-dirs=*)
+        MODULE_SKIPDIRS=${i#*=}
+        MODULE_SKIPDIRS=${MODULE_SKIPDIRS//,/ }
+        yetus_debug "Setting skipdirs to ${MODULE_SKIPDIRS}"
+      ;;
       --skip-system-plugins)
         LOAD_SYSTEM_PLUGINS=false
       ;;
@@ -1007,50 +1020,28 @@ function parse_args
   GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
 }
 
-## @description  Locate the pom.xml file for a given directory
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       directory containing the pom.xml. Nothing returned if not found.
-function find_pomxml_dir
-{
-  local dir=$1
-
-  yetus_debug "Find pom.xml dir for: ${dir}"
-
-  while builtin true; do
-    if [[ -f "${dir}/pom.xml" ]];then
-      echo "${dir}"
-      yetus_debug "Found: ${dir}"
-      return
-    elif [[ ${dir} == "." ]]; then
-      yetus_error "ERROR: pom.xml is not found. Make sure the target is a Maven-based project."
-      return
-    else
-      dir=$(dirname "${dir}")
-    fi
-  done
-}
-
-## @description  Locate the build.xml file for a given directory
+## @description  Locate the build file for a given directory
 ## @audience     private
 ## @stability    stable
 ## @replaceable  no
-## @return       directory containing the build.xml. Nothing returned if not found.
-function find_buildxml_dir
+## @return       directory containing the buildfile. Nothing returned if not found.
+## @params       buildfile
+## @params       directory
+function find_buildfile_dir
 {
-  local dir=$1
+  local buildfile=$1
+  local dir=$2
 
-  yetus_debug "Find build.xml dir for: ${dir}"
+  yetus_debug "Find ${buildfile} dir for: ${dir}"
 
   while builtin true; do
-    if [[ -f "${dir}/build.xml" ]];then
+    if [[ -f "${dir}/${buildfile}" ]];then
       echo "${dir}"
       yetus_debug "Found: ${dir}"
-      return
+      return 0
     elif [[ ${dir} == "." ]]; then
-      yetus_error "ERROR: build.xml is not found. Make sure the target is a Ant-based project."
-      return
+      yetus_debug "ERROR: ${buildfile} is not found."
+      return 1
     else
       dir=$(dirname "${dir}")
     fi
@@ -1073,6 +1064,42 @@ function find_changed_files
     /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
 }
 
+## @description Check for directories to skip during
+## @description changed module calcuation
+## @audience    private
+## @stability   stable
+## @replaceable no
+## @params      directory
+## @returns     0 for use
+## @returns     1 for skip
+function module_skipdir
+{
+  local dir=${1}
+  local i
+
+  yetus_debug "Checking skipdirs for ${dir}"
+
+  if [[ -z ${MODULE_SKIPDIRS} ]]; then
+    yetus_debug "Skipping skipdirs"
+    return 0
+  fi
+
+  while builtin true; do
+    for i in ${MODULE_SKIPDIRS}; do
+      if [[ ${dir} = "${i}" ]];then
+        yetus_debug "Found a skip: ${dir}"
+        return 1
+      fi
+    done
+    if [[ ${dir} == "." ]]; then
+      return 0
+    else
+      dir=$(dirname "${dir}")
+      yetus_debug "Trying to skip: ${dir}"
+    fi
+  done
+}
+
 ## @description  Find the modules of the build that ${PATCH_DIR}/patch modifies
 ## @audience     private
 ## @stability    stable
@@ -1082,60 +1109,111 @@ function find_changed_modules
 {
   local i
   local changed_dirs
-  local pomdirs
-  local pomdir
+  local builddirs
+  local builddir
   local module
-  local pommods
+  local buildmods
+  local prev_builddir
+  local i=1
+  local dir
+  local buildfile
+
+  case ${BUILDTOOL} in
+    maven)
+      buildfile=pom.xml
+    ;;
+    ant)
+      buildfile=build.xml
+    ;;
+    *)
+      yetus_error "ERROR: Unsupported build tool."
+      output_to_console 1
+      output_to_bugsystem 1
+      cleanup_and_exit 1
+    ;;
+  esac
 
   changed_dirs=$(for i in ${CHANGED_FILES}; do dirname "${i}"; done | sort -u)
 
   # Now find all the modules that were changed
   for i in ${changed_dirs}; do
-    case ${BUILDTOOL} in
-      maven)
-        #shellcheck disable=SC2086
-        pomdir=$(find_pomxml_dir ${i})
-        if [[ -z ${pomdir} ]]; then
-          output_to_console 1
-          output_to_bugsystem 1
-          cleanup_and_exit 1
-        fi
-        pomdirs="${pomdirs} ${pomdir}"
-      ;;
-      ant)
-        #shellcheck disable=SC2086
-        pomdir=$(find_buildxml_dir ${i})
-        if [[ -z ${pomdir} ]]; then
-          output_to_console 1
-          output_to_bugsystem 1
-          cleanup_and_exit 1
-        fi
-        pomdirs="${pomdirs} ${pomdir}"
-      ;;
-      *)
-        yetus_error "ERROR: Unsupported build tool."
-        output_to_console 1
-        output_to_bugsystem 1
-        cleanup_and_exit 1
-      ;;
-    esac
+
+    module_skipdir "${i}"
+    if [[ $? != 0 ]]; then
+      continue
+    fi
+
+    builddir=$(find_buildfile_dir ${buildfile} "${i}")
+    if [[ -z ${builddir} ]]; then
+      yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based
project."
+      output_to_console 1
+      output_to_bugsystem 1
+      cleanup_and_exit 1
+    fi
+    builddirs="${builddirs} ${builddir}"
   done
 
   #shellcheck disable=SC2086,SC2034
-  CHANGED_UNFILTERED_MODULES=$(echo ${pomdirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+  CHANGED_UNFILTERED_MODULES=$(echo ${builddirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort
-u)
+  #shellcheck disable=SC2086,SC2116
+  CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES})
+
 
-  if [[ ${BUILDTOOL} == maven ]]; then
+  if [[ ${BUILDTOOL} = maven
+    && ${QETESTMODE} = false ]]; then
     # Filter out modules without code
-    for module in ${pomdirs}; do
+    for module in ${builddirs}; do
       ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
       if [[ "$?" != 0 ]]; then
-        pommods="${pommods} ${module}"
+        buildmods="${buildmods} ${module}"
       fi
     done
+  elif [[ ${QETESTMODE} = true ]]; then
+    buildmods=${builddirs}
   fi
 
   #shellcheck disable=SC2086,SC2034
-  CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+  CHANGED_MODULES=$(echo ${buildmods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+
+  # turn it back into a list so that anyone printing doesn't
+  # generate multiline output
+  #shellcheck disable=SC2086,SC2116
+  CHANGED_MODULES=$(echo ${CHANGED_MODULES})
+
+  yetus_debug "Locate the union of ${CHANGED_MODULES}"
+  # shellcheck disable=SC2086
+  count=$(echo ${CHANGED_MODULES} | wc -w)
+  if [[ ${count} -lt 2 ]]; then
+    yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES}"
+    # shellcheck disable=SC2034
+    CHANGED_UNION_MODULES=${CHANGED_MODULES}
+    return
+  fi
+
+  i=1
+  while [[ ${i} -lt 100 ]]
+  do
+    module=$(echo "${CHANGED_MODULES}" | tr ' ' '\n' | cut -f1-${i} -d/ | uniq)
+    count=$(echo "${module}" | wc -w)
+    if [[ ${count} -eq 1
+      && -f ${module}/${buildfile} ]]; then
+      prev_builddir=${module}
+    elif [[ ${count} -gt 1 ]]; then
+      builddir=${prev_builddir}
+      break
+    fi
+    ((i=i+1))
+  done
+
+  if [[ -z ${builddir} ]]; then
+    builddir="."
+  fi
+
+  yetus_debug "Finding union of ${builddir}"
+  builddir=$(find_buildfile_dir ${buildfile} "${builddir}" || true)
+
+  #shellcheck disable=SC2034
+  CHANGED_UNION_MODULES="${builddir}"
 }
 
 ## @description  git checkout the appropriate branch to test.  Additionally, this calls


Mime
View raw message