Return-Path: X-Original-To: apmail-yetus-commits-archive@minotaur.apache.org Delivered-To: apmail-yetus-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6125419B26 for ; Thu, 14 Apr 2016 07:24:57 +0000 (UTC) Received: (qmail 21701 invoked by uid 500); 14 Apr 2016 07:24:57 -0000 Delivered-To: apmail-yetus-commits-archive@yetus.apache.org Received: (qmail 21624 invoked by uid 500); 14 Apr 2016 07:24:57 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 21484 invoked by uid 99); 14 Apr 2016 07:24:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Apr 2016 07:24:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DBB85E0B49; Thu, 14 Apr 2016 07:24:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: commits@yetus.apache.org Date: Thu, 14 Apr 2016 07:25:05 -0000 Message-Id: <26dbb5761b5a4e3ab842e95fd4728015@git.apache.org> In-Reply-To: <7eb7171b238d426fb7f8f3262a4578d8@git.apache.org> References: <7eb7171b238d426fb7f8f3262a4578d8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/20] yetus git commit: YETUS-349. core engine changes for build driver YETUS-349. core engine changes for build driver Signed-off-by: Allen Wittenauer Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/38d3a695 Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/38d3a695 Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/38d3a695 Branch: refs/heads/YETUS-156 Commit: 38d3a6958d8ff1aa28bc80be351c11e5400272c4 Parents: 9e55702 Author: Allen Wittenauer Authored: Tue Mar 29 16:30:27 2016 -0700 Committer: Allen Wittenauer Committed: Thu Apr 14 00:24:45 2016 -0700 ---------------------------------------------------------------------- precommit/core.d/01-common.sh | 22 +++++ precommit/test-patch.sh | 163 ++++++++++++++++++++++++------------- 2 files changed, 130 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/38d3a695/precommit/core.d/01-common.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh index 0661f6e..f31d613 100755 --- a/precommit/core.d/01-common.sh +++ b/precommit/core.d/01-common.sh @@ -536,6 +536,9 @@ function verify_command yetus_error "executable for '${cmd_name}' was not specified." return 1 fi + if [[ ! "${cmd_path}" =~ / ]]; then + cmd_path=$(command -v "${cmd_path}") + fi if [[ ! -f ${cmd_path} ]]; then yetus_error "executable '${cmd_path}' for '${cmd_name}' does not exist." return 1 @@ -546,3 +549,22 @@ function verify_command fi return 0 } + +## @description Faster dirname, given the assumption that +## @description dirs are always absolute (e.g., start with /) +## @description DO NOT USE with relative paths or where +## @description assumption may not be valid! +## @audience private +## @stability evolving +## @replaceable no +## @param fileobj +function faster_dirname +{ + declare o=$1 + + if [[ "${o}" =~ / ]]; then + echo "${o%/*}" + else + echo . + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/38d3a695/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index e0dbc29..e01ee87 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -86,6 +86,9 @@ function setup_defaults CHANGED_UNION_MODULES="" REEXECED=false RESETREPO=false + BUILDMODE=patch + # shellcheck disable=SC2034 + BUILDMODEMSG="The patch" ISSUE="" TIMER=$(date +"%s") BUILDTOOL=maven @@ -659,10 +662,14 @@ function yetus_usage jdktlist=$(echo ${JDK_TEST_LIST}) jdktlist=${jdktlist// /,} - echo "test-patch.sh [OPTIONS] patch" - echo "" - echo "Where:" - echo " patch is a file, URL, or bugsystem-compatible location of the patch file" + if [[ "${BUILDMODE}" = patch ]]; then + echo "${BINNAME} [OPTIONS] patch" + echo "" + echo "Where:" + echo " patch is a file, URL, or bugsystem-compatible location of the patch file" + else + echo "${BINNAME} [OPTIONS]" + fi echo "" echo "Options:" echo "" @@ -676,6 +683,7 @@ function yetus_usage yetus_add_option "--contrib-guide=" "URL to point new users towards project conventions. (default: ${PATCH_NAMING_RULE} )" yetus_add_option "--debug" "If set, then output some extra stuff to stderr" yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted changes" + yetus_add_option "--empty-patch" "Create a summary of the current source tree" yetus_add_option "--java-home=" "Set JAVA_HOME (In Docker mode, this should be local to the image)" yetus_add_option "--linecomments=" "Only write line comments to this comma delimited list (defaults to bugcomments)" yetus_add_option "--list-plugins" "List all installed plug-ins and then exit" @@ -800,6 +808,11 @@ function parse_args --instance=*) INSTANCE=${i#*=} ;; + --empty-patch) + BUILDMODE=full + # shellcheck disable=SC2034 + BUILDMODEMSG="The source tree" + ;; --java-home=*) JAVA_HOME=${i#*=} ;; @@ -890,7 +903,8 @@ function parse_args docker_parse_args "$@" - if [[ -z "${PATCH_OR_ISSUE}" ]]; then + if [[ -z "${PATCH_OR_ISSUE}" + && "${BUILDMODE}" = patch ]]; then yetus_usage exit 1 fi @@ -1009,7 +1023,7 @@ function find_buildfile_dir yetus_debug "ERROR: ${buildfile} is not found." return 1 else - dir=$(dirname "${dir}") + dir=$(faster_dirname "${dir}") fi done } @@ -1022,17 +1036,29 @@ function find_buildfile_dir function find_changed_files { declare line + declare oldifs - # get a list of all of the files that have been changed, - # except for /dev/null (which would be present for new files). - # Additionally, remove any a/ b/ patterns at the front of the patch filenames. - # shellcheck disable=SC2016 - while read -r line; do - CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}") - done < <( - ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}} - /^diff --git / { p($3); p($4) } - /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + case "${BUILDMODE}" in + full) + echo "Building a list of all files in the source tree" + oldifs=${IFS} + IFS=$'\n' + CHANGED_FILES=($(git ls-files)) + IFS=${oldifs} + ;; + patch) + # get a list of all of the files that have been changed, + # except for /dev/null (which would be present for new files). + # Additionally, remove any a/ b/ patterns at the front of the patch filenames. + # shellcheck disable=SC2016 + while read -r line; do + CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}") + done < <( + ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}} + /^diff --git / { p($3); p($4) } + /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + ;; + esac } ## @description Check for directories to skip during @@ -1065,7 +1091,7 @@ function module_skipdir if [[ ${dir} == "." || ${dir} == "/" ]]; then return 0 else - dir=$(dirname "${dir}") + dir=$(faster_dirname "${dir}") yetus_debug "Trying to skip: ${dir}" fi done @@ -1102,8 +1128,15 @@ function find_changed_modules if [[ -z ${buildfile} ]]; then tmpmods=(".") else + # Now find all the modules that were changed for i in "${CHANGED_FILES[@]}"; do + + # TODO: optimize this + if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then + continue + fi + dirt=$(dirname "${i}") module_skipdir "${dirt}" @@ -1272,8 +1305,10 @@ function git_checkout currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD) if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then - echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." - echo "WARNING: Continuing anyway..." + if [[ "${BUILDMODE}" = patch ]]; then + echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." + echo "WARNING: Continuing anyway..." + fi PATCH_BRANCH=${currentbranch} fi fi @@ -1398,8 +1433,11 @@ function determine_issue ## @replaceable no function determine_needed_tests { - local i - local plugin + declare i + declare plugin + + big_console_header "Determining needed tests" + echo "(Depending upon input size and number of plug-ins, this may take a while)" for i in "${CHANGED_FILES[@]}"; do yetus_debug "Determining needed tests for ${i}" @@ -1592,7 +1630,7 @@ function check_reexec fi done - if [[ ${copy} == true ]]; then + if [[ ${copy} == true && "${BUILDMODE}" != full ]]; then big_console_header "precommit patch detected" if [[ ${RESETREPO} == false ]]; then @@ -1709,8 +1747,10 @@ function modules_messages declare statusjdk declare multijdkmode=false - if [[ ${repostatus} == branch ]]; then + if [[ "${repostatus}" == branch ]]; then repo=${PATCH_BRANCH} + elif [[ "${BUILDMODE}" == full ]]; then + repo="the source" else repo="the patch" fi @@ -2351,12 +2391,16 @@ function generic_calcdiff_status ((samepatch=numpatch-addpatch)) ((fixedpatch=numbranch-numpatch+addpatch)) - printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \ - "${addpatch}" \ - "${samepatch}" \ - "${fixedpatch}" \ - "${numpatch}" \ - "${numbranch}" + if [[ "${BUILDMODE}" = full ]]; then + printf "has %i issues." "${addpatch}" + else + printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \ + "${addpatch}" \ + "${samepatch}" \ + "${fixedpatch}" \ + "${numpatch}" \ + "${numbranch}" + fi } ## @description Helper routine for plugins to ask projects, etc @@ -2562,7 +2606,7 @@ function generic_post_handler return 0 fi - big_console_header "Patch ${testtype} verification" + big_console_header "${testtype} verification: ${BUILDMODE}" for jdkindex in ${JDK_DIR_LIST}; do if [[ ${multijdkmode} == true ]]; then @@ -2683,9 +2727,9 @@ function compile fi if [[ ${codebase} = "branch" ]]; then - big_console_header "Pre-patch ${PATCH_BRANCH} compilation" + big_console_header "${PATCH_BRANCH} compilation: pre-patch" else - big_console_header "Patch compilation" + big_console_header "${PATCH_BRANCH} compilation: ${BUILDMODE}" fi yetus_debug "Is JVM Required? ${JVM_REQUIRED}" @@ -2858,26 +2902,33 @@ function initialize echo "Modes: ${EXEC_MODES}" - locate_patch + if [[ "${BUILDMODE}" = patch ]]; then + locate_patch + + # from here on out, we'll be in ${BASEDIR} for cwd + # plugins need to pushd/popd if they change. + git_checkout - # from here on out, we'll be in ${BASEDIR} for cwd - # plugins need to pushd/popd if they change. - git_checkout + determine_issue + if [[ "${ISSUE}" == 'Unknown' ]]; then + echo "Testing patch on ${PATCH_BRANCH}." + else + echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." + fi + + patchfile_dryrun_driver "${PATCH_DIR}/patch" + if [[ $? != 0 ]]; then + ((RESULT = RESULT + 1)) + yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}." + add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi - determine_issue - if [[ "${ISSUE}" == 'Unknown' ]]; then - echo "Testing patch on ${PATCH_BRANCH}." else - echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." - fi - patchfile_dryrun_driver "${PATCH_DIR}/patch" - if [[ $? != 0 ]]; then - ((RESULT = RESULT + 1)) - yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}." - add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." - bugsystem_finalreport 1 - cleanup_and_exit 1 + git_checkout + fi find_changed_files @@ -2948,17 +2999,19 @@ initialize "$@" prechecks -patchfiletests -((RESULT=RESULT+$?)) +if [[ "${BUILDMODE}" = patch ]]; then + patchfiletests + ((RESULT=RESULT+$?)) -compile_cycle branch -((RESULT=RESULT+$?)) + compile_cycle branch + ((RESULT=RESULT+$?)) -distclean + distclean -apply_patch_file + apply_patch_file -compute_gitdiff + compute_gitdiff +fi compile_cycle patch ((RESULT=RESULT+$?))