hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject hadoop git commit: HADOOP-12198. hadoop patches that hit multiple modules need to build at the union (aw)
Date Tue, 21 Jul 2015 23:51:17 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-12111 a3a4578cc -> 27a2328c5


HADOOP-12198. hadoop patches that hit multiple modules need to build at the union (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27a2328c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27a2328c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27a2328c

Branch: refs/heads/HADOOP-12111
Commit: 27a2328c5dea952704a1a8807ae0c7fb44fa84f4
Parents: a3a4578
Author: Allen Wittenauer <aw@apache.org>
Authored: Tue Jul 21 16:51:03 2015 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Tue Jul 21 16:51:03 2015 -0700

----------------------------------------------------------------------
 dev-support/personality/hadoop.sh      | 237 +++++++++++++++++-----------
 dev-support/test-patch.d/checkstyle.sh |   2 +-
 dev-support/test-patch.d/findbugs.sh   |   2 +-
 dev-support/test-patch.sh              |  19 ++-
 4 files changed, 160 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
index 7722afb..3d6e3fa 100755
--- a/dev-support/personality/hadoop.sh
+++ b/dev-support/personality/hadoop.sh
@@ -27,63 +27,107 @@ HADOOP_MODULES=""
 
 function hadoop_module_manipulation
 {
-  local need_common=0
+  local startingmodules=${1:-normal}
   local module
   local hdfs_modules
   local ordered_modules
   local tools_modules
-  local passed_modules=${CHANGED_MODULES}
+  local passed_modules
+  local flags
+
+  yetus_debug "hmm in: ${startingmodules}"
 
-  yetus_debug "hmm: starting list: ${passed_modules}"
+  if [[ ${startingmodules} = normal ]]; then
+    startingmodules=${CHANGED_MODULES}
+  elif  [[ ${startingmodules} = union ]]; then
+    startingmodules=${CHANGED_UNION_MODULES}
+  fi
 
-  # if one of our modules is ., then shortcut:
-  # ignore the rest and just set it to everything.
-  if [[ ${CHANGED_MODULES} == ' . ' ]]; then
-    HADOOP_MODULES='.'
+  yetus_debug "hmm expanded to: ${startingmodules}"
+
+  if [[ ${startingmodules} = "." ]]; then
+    yetus_debug "hmm shortcut since ."
+    HADOOP_MODULES=.
     return
   fi
 
-  # ${CHANGED_MODULES} is already sorted and uniq'd.
+  # ${startingmodules} is already sorted and uniq'd.
   # let's remove child modules if we're going to
-  # touch their parent
-  for module in ${CHANGED_MODULES}; do
+  # touch their parent.
+  passed_modules=${startingmodules}
+  for module in ${startingmodules}; do
     yetus_debug "Stripping ${module}"
     # shellcheck disable=SC2086
     passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ )
   done
 
+  yetus_debug "hmm pre-ordering: ${startingmodules}"
+
+  # yarn will almost always be after common in the sort order
+  # so really just need to make sure that common comes before
+  # everything else and tools comes last
+
   for module in ${passed_modules}; do
     yetus_debug "Personality ordering ${module}"
-    if [[ ${module} == hadoop-hdfs-project* ]]; then
+    if [[ ${module} = "." ]]; then
+      HADOOP_MODULES=.
+      break
+    fi
+
+    if [[ ${module} = hadoop-hdfs-project* ]]; then
       hdfs_modules="${hdfs_modules} ${module}"
-      need_common=1
-    elif [[ ${module} == hadoop-common-project/hadoop-common
-      || ${module} == hadoop-common-project ]]; then
+    elif [[ ${module} = hadoop-common-project/hadoop-common
+      || ${module} = hadoop-common-project ]]; then
       ordered_modules="${ordered_modules} ${module}"
-      building_common=1
-    elif [[ ${module} == hadoop-tools* ]]; then
+    elif [[ ${module} = hadoop-tools* ]]; then
       tools_modules="${tools_modules} ${module}"
     else
       ordered_modules="${ordered_modules} ${module}"
     fi
   done
 
-  ordered_modules="${ordered_modules} ${hdfs_modules} ${tools_modules}"
+  HADOOP_MODULES="${ordered_modules} ${hdfs_modules} ${tools_modules}"
+
+  yetus_debug "hmm out: ${HADOOP_MODULES}"
+}
+
+function hadoop_unittest_prereqs
+{
+  local need_common=0
+  local building_common=0
+  local module
+  local flags
+  local fn
+
+  for module in ${HADOOP_MODULES}; do
+    if [[ ${module} = hadoop-hdfs-project* ]]; then
+      need_common=1
+    elif [[ ${module} = hadoop-common-project/hadoop-common
+      || ${module} = hadoop-common-project ]]; then
+      building_common=1
+    fi
+  done
 
   if [[ ${need_common} -eq 1
       && ${building_common} -eq 0 ]]; then
-      ordered_modules="hadoop-common-project/hadoop-common ${ordered_modules}"
+    echo "unit test pre-reqs:"
+    module="hadoop-common-project/hadoop-common"
+    fn=$(module_file_fragment "${module}")
+    flags=$(hadoop_native_flags)
+    pushd "${BASEDIR}/${module}" >/dev/null
+    # shellcheck disable=SC2086
+    echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
+      "${MVN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
+    popd >/dev/null
   fi
-
-  yetus_debug "hmm: ${ordered_modules}"
-  HADOOP_MODULES=${ordered_modules}
 }
 
-function hadoop_javac_ordering
+function hadoop_native_flags
 {
-  local special=$1
-  local ordered_modules
-  local module
+
+  if [[ ${BUILD_NATIVE} != true ]]; then
+    return
+  fi
 
   # Based upon HADOOP-11937
   #
@@ -97,46 +141,45 @@ function hadoop_javac_ordering
   #   e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
   #
 
-  for module in ${HADOOP_MODULES}; do
-    if [[ ${JENKINS} == true
-        && ${DOCKERSUPPORT} == false ]]; then
+  # current build servers are pretty limited in
+  # what they support
+  if [[ ${JENKINS} = true
+      && ${DOCKERSUPPORT} = false ]]; then
+    # shellcheck disable=SC2086
+    echo -Pnative \
+      -Drequire.snappy -Drequire.openssl -Drequire.fuse \
+      -Drequire.test.libhadoop
+    return
+  fi
+
+  case ${OSTYPE} in
+    Linux)
       # shellcheck disable=SC2086
-      personality_enqueue_module "${module}" ${special} \
-        -Pnative \
+      echo -Pnative -Drequire.libwebhdfs \
         -Drequire.snappy -Drequire.openssl -Drequire.fuse \
         -Drequire.test.libhadoop
-    else
-      case ${OSTYPE} in
-        Linux)
-          # shellcheck disable=SC2086
-          personality_enqueue_module ${module} ${special} \
-            -Pnative -Drequire.libwebhdfs \
-            -Drequire.snappy -Drequire.openssl -Drequire.fuse \
-            -Drequire.test.libhadoop
-        ;;
-        Darwin)
-          JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include
-          JANSSON_LIBRARY=/usr/local/opt/jansson/lib
-          export JANSSON_LIBRARY JANSSON_INCLUDE_DIR
-          # shellcheck disable=SC2086
-          personality_enqueue_module ${module} ${special} \
-          -Pnative -Drequire.snappy  \
-          -Drequire.openssl \
-            -Dopenssl.prefix=/usr/local/opt/openssl/ \
-            -Dopenssl.include=/usr/local/opt/openssl/include \
-            -Dopenssl.lib=/usr/local/opt/openssl/lib \
-          -Drequire.libwebhdfs -Drequire.test.libhadoop
-        ;;
-        *)
-          # shellcheck disable=SC2086
-          personality_enqueue_module ${module} ${special} \
-            -Pnative \
-            -Drequire.snappy -Drequire.openssl \
-            -Drequire.libwebhdfs -Drequire.test.libhadoop
-        ;;
-      esac
-    fi
-  done
+    ;;
+    Darwin)
+      JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include
+      JANSSON_LIBRARY=/usr/local/opt/jansson/lib
+      export JANSSON_LIBRARY JANSSON_INCLUDE_DIR
+      # shellcheck disable=SC2086
+      echo \
+      -Pnative -Drequire.snappy  \
+      -Drequire.openssl \
+        -Dopenssl.prefix=/usr/local/opt/openssl/ \
+        -Dopenssl.include=/usr/local/opt/openssl/include \
+        -Dopenssl.lib=/usr/local/opt/openssl/lib \
+      -Drequire.libwebhdfs -Drequire.test.libhadoop
+    ;;
+    *)
+      # shellcheck disable=SC2086
+      echo \
+        -Pnative \
+        -Drequire.snappy -Drequire.openssl \
+        -Drequire.libwebhdfs -Drequire.test.libhadoop
+    ;;
+  esac
 }
 
 function personality_modules
@@ -144,6 +187,9 @@ function personality_modules
   local repostatus=$1
   local testtype=$2
   local extra=""
+  local ordering="normal"
+  local needflags=false
+  local flags
   local fn
   local i
 
@@ -152,16 +198,29 @@ function personality_modules
   clear_personality_queue
 
   case ${testtype} in
+    asflicense)
+      # this is very fast and provides the full path if we do it from
+      # the root of the source
+      personality_enqueue_module .
+      return
+    ;;
+    checkstyle)
+      ordering="union"
+      extra="-DskipTests"
+    ;;
     javac)
-      if [[ ${BUILD_NATIVE} == true ]]; then
-        hadoop_module_manipulation
-        hadoop_javac_ordering -DskipTests
-        return
-      fi
+      ordering="union"
       extra="-DskipTests"
+      needflags=true
+
+      # if something in common changed, we build the whole world
+      if [[ ${CHANGED_MODULES} =~ hadoop-common ]]; then
+        yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
+        ordering="."
+      fi
       ;;
     javadoc)
-      if [[ ${repostatus} == patch ]]; then
+      if [[ ${repostatus} = patch ]]; then
         echo "javadoc pre-reqs:"
         for i in  hadoop-project \
           hadoop-common-project/hadoop-annotations; do
@@ -177,38 +236,34 @@ function personality_modules
     ;;
     mvninstall)
       extra="-DskipTests"
-      if [[ ${repostatus} == branch ]]; then
-        HADOOP_MODULES=.
-        hadoop_javac_ordering -DskipTests
-        return
+      if [[ ${repostatus} = branch ]]; then
+        ordering=.
       fi
       ;;
-    asflicense)
-      # this is very fast and provides the full path if we do it from
-      # the root of the source
-      personality_enqueue_module .
-      return
-    ;;
     unit)
-      if [[ ${TEST_PARALLEL} == "true" ]] ; then
-        extra="-Pparallel-tests"
-        if [[ -n ${TEST_THREADS:-} ]]; then
-          extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
-        fi
-      fi
-      if [[ ${BUILD_NATIVE} == true ]]; then
-        hadoop_module_manipulation
-        # shellcheck disable=SC2086
-        hadoop_javac_ordering ${extra}
-        return
-      fi
+      # As soon as HADOOP-11984 gets committed,
+      # this code should get uncommented
+      #if [[ ${TEST_PARALLEL} = "true" ]] ; then
+      #  extra="-Pparallel-tests"
+      #  if [[ -n ${TEST_THREADS:-} ]]; then
+      #    extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+      #  fi
+      #fi
+      needflags=true
+      hadoop_unittest_prereqs
     ;;
     *)
       extra="-DskipTests"
     ;;
   esac
 
-  hadoop_module_manipulation
+  if [[ ${needflags} = true ]]; then
+    flags=$(hadoop_native_flags)
+    extra="${extra} ${flags}"
+  fi
+
+  hadoop_module_manipulation ${ordering}
+
   for module in ${HADOOP_MODULES}; do
     # shellcheck disable=SC2086
     personality_enqueue_module ${module} ${extra}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
index 1fbe88e..f4bf79d 100755
--- a/dev-support/test-patch.d/checkstyle.sh
+++ b/dev-support/test-patch.d/checkstyle.sh
@@ -64,7 +64,7 @@ function checkstyle_runner
 
     case ${BUILDTOOL} in
       maven)
-        cmd="${MVN} ${MAVEN_ARGS[*]} clean test \
+        cmd="${MVN} ${MAVEN_ARGS[*]} \
            checkstyle:checkstyle \
           -Dcheckstyle.consoleOutput=true \
           ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.d/findbugs.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh
index 2fe2339..1d7118b 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -90,7 +90,7 @@ function findbugs_runner
   personality_modules "${name}" findbugs
   case ${BUILDTOOL} in
     maven)
-      modules_workers "${name}" findbugs clean test findbugs:findbugs
+      modules_workers "${name}" findbugs test-compile findbugs:findbugs
     ;;
     ant)
       modules_workers "${name}" findbugs findbugs

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 25e8649..4dc9cfd 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1175,7 +1175,6 @@ function find_changed_modules
   #shellcheck disable=SC2086,SC2116
   CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES})
 
-
   if [[ ${BUILDTOOL} = maven
     && ${QETESTMODE} = false ]]; then
     # Filter out modules without code
@@ -2229,7 +2228,7 @@ function precheck_javac
     personality_modules branch javac
     case ${BUILDTOOL} in
       maven)
-        modules_workers branch javac clean compile
+        modules_workers branch javac clean test-compile
       ;;
       ant)
         modules_workers branch javac
@@ -2530,7 +2529,7 @@ function check_patch_javac
 
     case ${BUILDTOOL} in
       maven)
-        modules_workers patch javac clean compile
+        modules_workers patch javac clean test-compile
       ;;
       ant)
         modules_workers patch javac
@@ -2842,7 +2841,7 @@ function check_mvninstall
   fi
 
   personality_modules patch mvninstall
-  modules_workers patch mvninstall install -Dmaven.javadoc.skip=true
+  modules_workers patch mvninstall clean install -Dmaven.javadoc.skip=true
   result=$?
   modules_messages patch mvninstall true
   if [[ ${result} != 0 ]]; then
@@ -3276,7 +3275,8 @@ function postapply
 
   ((RESULT = RESULT + retval))
 
-  for routine in check_patch_javadoc check_site
+  # shellcheck disable=SC2043
+  for routine in check_site
   do
     verify_patchdir_still_exists
     yetus_debug "Running ${routine}"
@@ -3305,8 +3305,13 @@ function postinstall
   local plugin
 
   verify_patchdir_still_exists
-  check_mvn_eclipse
-  (( RESULT = RESULT + $? ))
+  for routine in check_patch_javadoc check_mvn_eclipse
+  do
+    verify_patchdir_still_exists
+    yetus_debug "Running ${routine}"
+    ${routine}
+    (( RESULT = RESULT + $? ))
+  done
 
   for plugin in ${PLUGINS}; do
     verify_patchdir_still_exists


Mime
View raw message