Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E45BB18611 for ; Tue, 21 Jul 2015 23:51:17 +0000 (UTC) Received: (qmail 27071 invoked by uid 500); 21 Jul 2015 23:51:17 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 27006 invoked by uid 500); 21 Jul 2015 23:51:17 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 26997 invoked by uid 99); 21 Jul 2015 23:51:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Jul 2015 23:51:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0A4BFE03E8; Tue, 21 Jul 2015 23:51:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: common-commits@hadoop.apache.org Message-Id: <4ba184556f734368b1dba10e7c62d441@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-12198. hadoop patches that hit multiple modules need to build at the union (aw) Date: Tue, 21 Jul 2015 23:51:17 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/HADOOP-12111 a3a4578cc -> 27a2328c5 HADOOP-12198. hadoop patches that hit multiple modules need to build at the union (aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27a2328c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27a2328c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27a2328c Branch: refs/heads/HADOOP-12111 Commit: 27a2328c5dea952704a1a8807ae0c7fb44fa84f4 Parents: a3a4578 Author: Allen Wittenauer Authored: Tue Jul 21 16:51:03 2015 -0700 Committer: Allen Wittenauer Committed: Tue Jul 21 16:51:03 2015 -0700 ---------------------------------------------------------------------- dev-support/personality/hadoop.sh | 237 +++++++++++++++++----------- dev-support/test-patch.d/checkstyle.sh | 2 +- dev-support/test-patch.d/findbugs.sh | 2 +- dev-support/test-patch.sh | 19 ++- 4 files changed, 160 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh index 7722afb..3d6e3fa 100755 --- a/dev-support/personality/hadoop.sh +++ b/dev-support/personality/hadoop.sh @@ -27,63 +27,107 @@ HADOOP_MODULES="" function hadoop_module_manipulation { - local need_common=0 + local startingmodules=${1:-normal} local module local hdfs_modules local ordered_modules local tools_modules - local passed_modules=${CHANGED_MODULES} + local passed_modules + local flags + + yetus_debug "hmm in: ${startingmodules}" - yetus_debug "hmm: starting list: ${passed_modules}" + if [[ ${startingmodules} = normal ]]; then + startingmodules=${CHANGED_MODULES} + elif [[ ${startingmodules} = union ]]; then + startingmodules=${CHANGED_UNION_MODULES} + fi - # if one of our modules is ., then shortcut: - # ignore the rest and just set it to everything. - if [[ ${CHANGED_MODULES} == ' . ' ]]; then - HADOOP_MODULES='.' + yetus_debug "hmm expanded to: ${startingmodules}" + + if [[ ${startingmodules} = "." ]]; then + yetus_debug "hmm shortcut since ." + HADOOP_MODULES=. return fi - # ${CHANGED_MODULES} is already sorted and uniq'd. + # ${startingmodules} is already sorted and uniq'd. # let's remove child modules if we're going to - # touch their parent - for module in ${CHANGED_MODULES}; do + # touch their parent. + passed_modules=${startingmodules} + for module in ${startingmodules}; do yetus_debug "Stripping ${module}" # shellcheck disable=SC2086 passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ ) done + yetus_debug "hmm pre-ordering: ${startingmodules}" + + # yarn will almost always be after common in the sort order + # so really just need to make sure that common comes before + # everything else and tools comes last + for module in ${passed_modules}; do yetus_debug "Personality ordering ${module}" - if [[ ${module} == hadoop-hdfs-project* ]]; then + if [[ ${module} = "." ]]; then + HADOOP_MODULES=. + break + fi + + if [[ ${module} = hadoop-hdfs-project* ]]; then hdfs_modules="${hdfs_modules} ${module}" - need_common=1 - elif [[ ${module} == hadoop-common-project/hadoop-common - || ${module} == hadoop-common-project ]]; then + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then ordered_modules="${ordered_modules} ${module}" - building_common=1 - elif [[ ${module} == hadoop-tools* ]]; then + elif [[ ${module} = hadoop-tools* ]]; then tools_modules="${tools_modules} ${module}" else ordered_modules="${ordered_modules} ${module}" fi done - ordered_modules="${ordered_modules} ${hdfs_modules} ${tools_modules}" + HADOOP_MODULES="${ordered_modules} ${hdfs_modules} ${tools_modules}" + + yetus_debug "hmm out: ${HADOOP_MODULES}" +} + +function hadoop_unittest_prereqs +{ + local need_common=0 + local building_common=0 + local module + local flags + local fn + + for module in ${HADOOP_MODULES}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + fi + done if [[ ${need_common} -eq 1 && ${building_common} -eq 0 ]]; then - ordered_modules="hadoop-common-project/hadoop-common ${ordered_modules}" + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags=$(hadoop_native_flags) + pushd "${BASEDIR}/${module}" >/dev/null + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MVN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null fi - - yetus_debug "hmm: ${ordered_modules}" - HADOOP_MODULES=${ordered_modules} } -function hadoop_javac_ordering +function hadoop_native_flags { - local special=$1 - local ordered_modules - local module + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi # Based upon HADOOP-11937 # @@ -97,46 +141,45 @@ function hadoop_javac_ordering # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 # - for module in ${HADOOP_MODULES}; do - if [[ ${JENKINS} == true - && ${DOCKERSUPPORT} == false ]]; then + # current build servers are pretty limited in + # what they support + if [[ ${JENKINS} = true + && ${DOCKERSUPPORT} = false ]]; then + # shellcheck disable=SC2086 + echo -Pnative \ + -Drequire.snappy -Drequire.openssl -Drequire.fuse \ + -Drequire.test.libhadoop + return + fi + + case ${OSTYPE} in + Linux) # shellcheck disable=SC2086 - personality_enqueue_module "${module}" ${special} \ - -Pnative \ + echo -Pnative -Drequire.libwebhdfs \ -Drequire.snappy -Drequire.openssl -Drequire.fuse \ -Drequire.test.libhadoop - else - case ${OSTYPE} in - Linux) - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${special} \ - -Pnative -Drequire.libwebhdfs \ - -Drequire.snappy -Drequire.openssl -Drequire.fuse \ - -Drequire.test.libhadoop - ;; - Darwin) - JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include - JANSSON_LIBRARY=/usr/local/opt/jansson/lib - export JANSSON_LIBRARY JANSSON_INCLUDE_DIR - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${special} \ - -Pnative -Drequire.snappy \ - -Drequire.openssl \ - -Dopenssl.prefix=/usr/local/opt/openssl/ \ - -Dopenssl.include=/usr/local/opt/openssl/include \ - -Dopenssl.lib=/usr/local/opt/openssl/lib \ - -Drequire.libwebhdfs -Drequire.test.libhadoop - ;; - *) - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${special} \ - -Pnative \ - -Drequire.snappy -Drequire.openssl \ - -Drequire.libwebhdfs -Drequire.test.libhadoop - ;; - esac - fi - done + ;; + Darwin) + JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include + JANSSON_LIBRARY=/usr/local/opt/jansson/lib + export JANSSON_LIBRARY JANSSON_INCLUDE_DIR + # shellcheck disable=SC2086 + echo \ + -Pnative -Drequire.snappy \ + -Drequire.openssl \ + -Dopenssl.prefix=/usr/local/opt/openssl/ \ + -Dopenssl.include=/usr/local/opt/openssl/include \ + -Dopenssl.lib=/usr/local/opt/openssl/lib \ + -Drequire.libwebhdfs -Drequire.test.libhadoop + ;; + *) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.snappy -Drequire.openssl \ + -Drequire.libwebhdfs -Drequire.test.libhadoop + ;; + esac } function personality_modules @@ -144,6 +187,9 @@ function personality_modules local repostatus=$1 local testtype=$2 local extra="" + local ordering="normal" + local needflags=false + local flags local fn local i @@ -152,16 +198,29 @@ function personality_modules clear_personality_queue case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; javac) - if [[ ${BUILD_NATIVE} == true ]]; then - hadoop_module_manipulation - hadoop_javac_ordering -DskipTests - return - fi + ordering="union" extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ ${CHANGED_MODULES} =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi ;; javadoc) - if [[ ${repostatus} == patch ]]; then + if [[ ${repostatus} = patch ]]; then echo "javadoc pre-reqs:" for i in hadoop-project \ hadoop-common-project/hadoop-annotations; do @@ -177,38 +236,34 @@ function personality_modules ;; mvninstall) extra="-DskipTests" - if [[ ${repostatus} == branch ]]; then - HADOOP_MODULES=. - hadoop_javac_ordering -DskipTests - return + if [[ ${repostatus} = branch ]]; then + ordering=. fi ;; - asflicense) - # this is very fast and provides the full path if we do it from - # the root of the source - personality_enqueue_module . - return - ;; unit) - if [[ ${TEST_PARALLEL} == "true" ]] ; then - extra="-Pparallel-tests" - if [[ -n ${TEST_THREADS:-} ]]; then - extra="${extra} -DtestsThreadCount=${TEST_THREADS}" - fi - fi - if [[ ${BUILD_NATIVE} == true ]]; then - hadoop_module_manipulation - # shellcheck disable=SC2086 - hadoop_javac_ordering ${extra} - return - fi + # As soon as HADOOP-11984 gets committed, + # this code should get uncommented + #if [[ ${TEST_PARALLEL} = "true" ]] ; then + # extra="-Pparallel-tests" + # if [[ -n ${TEST_THREADS:-} ]]; then + # extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + # fi + #fi + needflags=true + hadoop_unittest_prereqs ;; *) extra="-DskipTests" ;; esac - hadoop_module_manipulation + if [[ ${needflags} = true ]]; then + flags=$(hadoop_native_flags) + extra="${extra} ${flags}" + fi + + hadoop_module_manipulation ${ordering} + for module in ${HADOOP_MODULES}; do # shellcheck disable=SC2086 personality_enqueue_module ${module} ${extra} http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh index 1fbe88e..f4bf79d 100755 --- a/dev-support/test-patch.d/checkstyle.sh +++ b/dev-support/test-patch.d/checkstyle.sh @@ -64,7 +64,7 @@ function checkstyle_runner case ${BUILDTOOL} in maven) - cmd="${MVN} ${MAVEN_ARGS[*]} clean test \ + cmd="${MVN} ${MAVEN_ARGS[*]} \ checkstyle:checkstyle \ -Dcheckstyle.consoleOutput=true \ ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.d/findbugs.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh index 2fe2339..1d7118b 100755 --- a/dev-support/test-patch.d/findbugs.sh +++ b/dev-support/test-patch.d/findbugs.sh @@ -90,7 +90,7 @@ function findbugs_runner personality_modules "${name}" findbugs case ${BUILDTOOL} in maven) - modules_workers "${name}" findbugs clean test findbugs:findbugs + modules_workers "${name}" findbugs test-compile findbugs:findbugs ;; ant) modules_workers "${name}" findbugs findbugs http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a2328c/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 25e8649..4dc9cfd 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -1175,7 +1175,6 @@ function find_changed_modules #shellcheck disable=SC2086,SC2116 CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES}) - if [[ ${BUILDTOOL} = maven && ${QETESTMODE} = false ]]; then # Filter out modules without code @@ -2229,7 +2228,7 @@ function precheck_javac personality_modules branch javac case ${BUILDTOOL} in maven) - modules_workers branch javac clean compile + modules_workers branch javac clean test-compile ;; ant) modules_workers branch javac @@ -2530,7 +2529,7 @@ function check_patch_javac case ${BUILDTOOL} in maven) - modules_workers patch javac clean compile + modules_workers patch javac clean test-compile ;; ant) modules_workers patch javac @@ -2842,7 +2841,7 @@ function check_mvninstall fi personality_modules patch mvninstall - modules_workers patch mvninstall install -Dmaven.javadoc.skip=true + modules_workers patch mvninstall clean install -Dmaven.javadoc.skip=true result=$? modules_messages patch mvninstall true if [[ ${result} != 0 ]]; then @@ -3276,7 +3275,8 @@ function postapply ((RESULT = RESULT + retval)) - for routine in check_patch_javadoc check_site + # shellcheck disable=SC2043 + for routine in check_site do verify_patchdir_still_exists yetus_debug "Running ${routine}" @@ -3305,8 +3305,13 @@ function postinstall local plugin verify_patchdir_still_exists - check_mvn_eclipse - (( RESULT = RESULT + $? )) + for routine in check_patch_javadoc check_mvn_eclipse + do + verify_patchdir_still_exists + yetus_debug "Running ${routine}" + ${routine} + (( RESULT = RESULT + $? )) + done for plugin in ${PLUGINS}; do verify_patchdir_still_exists