yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sek...@apache.org
Subject [11/20] yetus git commit: YETUS-156. [Umbrella] build driver
Date Wed, 11 May 2016 21:41:33 GMT
http://git-wip-us.apache.org/repos/asf/yetus/blob/612a0f2d/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index 0260c59..05a075e 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -25,6 +25,8 @@ fi
 
 this="${BASH_SOURCE-$0}"
 BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
 STARTINGDIR=$(pwd)
 USER_PARAMS=("$@")
 GLOBALTIMER=$(date +"%s")
@@ -91,6 +93,9 @@ function setup_defaults
   CHANGED_UNION_MODULES=""
   REEXECED=false
   RESETREPO=false
+  BUILDMODE=patch
+  # shellcheck disable=SC2034
+  BUILDMODEMSG="The patch"
   ISSUE=""
   TIMER=$(date +"%s")
   BUILDTOOL=maven
@@ -231,12 +236,15 @@ function add_header_line
 ## @return       Elapsed time display
 function add_vote_table
 {
-  local value=$1
-  local subsystem=$2
+  declare value=$1
+  declare subsystem=$2
   shift 2
 
-  local calctime
-  local -r elapsed=$(stop_clock)
+  declare calctime
+  # apparently shellcheck doesn't know about declare -r
+  #shellcheck disable=SC2155
+  declare -r elapsed=$(stop_clock)
+  declare filt
 
   yetus_debug "add_vote_table ${value} ${subsystem} ${*}"
 
@@ -246,6 +254,12 @@ function add_vote_table
     value="+1"
   fi
 
+  for filt in "${VOTE_FILTER[@]}"; do
+    if [[ "${subsystem}" = "${filt}" ]]; then
+      value=0
+    fi
+  done
+
   if [[ -z ${value} ]]; then
     # shellcheck disable=SC2034
     TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="|  | ${subsystem} | | ${*:-} |"
@@ -254,6 +268,10 @@ function add_vote_table
     TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |"
   fi
   ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1))
+
+  if [[ "${value}" = -1 ]]; then
+    ((RESULT = RESULT + 1))
+  fi
 }
 
 ## @description  Report the JVM version of the given directory
@@ -651,7 +669,6 @@ function relative_dir
 ## @replaceable  no
 function yetus_usage
 {
-
   declare bugsys
   declare jdktlist
 
@@ -665,10 +682,14 @@ function yetus_usage
   jdktlist=$(echo ${JDK_TEST_LIST})
   jdktlist=${jdktlist// /,}
 
-  echo "test-patch.sh [OPTIONS] patch"
-  echo ""
-  echo "Where:"
-  echo "  patch is a file, URL, or bugsystem-compatible location of the patch file"
+  if [[ "${BUILDMODE}" = patch ]]; then
+    echo "${BINNAME} [OPTIONS] patch"
+    echo ""
+    echo "Where:"
+    echo "  patch is a file, URL, or bugsystem-compatible location of the patch file"
+  else
+    echo "${BINNAME} [OPTIONS]"
+  fi
   echo ""
   echo "Options:"
   echo ""
@@ -682,6 +703,7 @@ function yetus_usage
   yetus_add_option "--contrib-guide=<url>" "URL to point new users towards project
conventions. (default: ${PATCH_NAMING_RULE} )"
   yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
   yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted
changes"
+  yetus_add_option "--empty-patch" "Create a summary of the current source tree"
   yetus_add_option "--java-home=<path>" "Set JAVA_HOME (In Docker mode, this should
be local to the image)"
   yetus_add_option "--linecomments=<bug>" "Only write line comments to this comma delimited
list (defaults to bugcomments)"
   yetus_add_option "--list-plugins" "List all installed plug-ins and then exit"
@@ -700,6 +722,7 @@ function yetus_usage
   yetus_add_option "--summarize=<bool>" "Allow tests to summarize results"
   yetus_add_option "--test-parallel=<bool>" "Run multiple tests in parallel (default
false in developer mode, true in Jenkins mode)"
   yetus_add_option "--test-threads=<int>" "Number of tests to run in parallel (default
defined in ${PROJECT_NAME} build)"
+  yetus_add_option "--tests-filter=<list>" "Lists of tests to turn failures into warnings"
   yetus_add_option "--user-plugins=<dir>" "A directory of user provided plugins. see
test-patch.d for examples (default empty)"
   yetus_add_option "--version" "Print release version information and exit"
 
@@ -725,6 +748,7 @@ function yetus_usage
   yetus_add_option "--build-url=<url>" "Set the build location web page (Default: '${BUILD_URL}')"
   yetus_add_option "--build-url-console=<location>" "Location relative to --build-url
of the console (Default: '${BUILD_URL_CONSOLE}')"
   yetus_add_option "--build-url-patchdir=<location>" "Location relative to --build-url
of the --patch-dir (Default: '${BUILD_URL_ARTIFACTS}')"
+  yetus_add_option "--console-report-file=<file>" "Save the final console-based report
to a file in addition to the screen"
   yetus_add_option "--console-urls" "Use the build URL instead of path on the console report"
   yetus_add_option "--instance=<string>" "Parallel execution identifier string"
   yetus_add_option "--jenkins" "Enable Jenkins-specifc handling (auto: --robot)"
@@ -789,6 +813,9 @@ function parse_args
         # shellcheck disable=SC2034
         BUILD_URL_CONSOLE=${i#*=}
       ;;
+      --console-report-file=*)
+        CONSOLE_REPORT_FILE=${i#*=}
+      ;;
       --console-urls)
         # shellcheck disable=SC2034
         CONSOLE_USE_BUILD_URL=true
@@ -802,6 +829,11 @@ function parse_args
       --instance=*)
         INSTANCE=${i#*=}
       ;;
+      --empty-patch)
+        BUILDMODE=full
+        # shellcheck disable=SC2034
+        BUILDMODEMSG="The source tree"
+      ;;
       --java-home=*)
         JAVA_HOME=${i#*=}
       ;;
@@ -866,6 +898,9 @@ function parse_args
         # shellcheck disable=SC2034
         TEST_THREADS=${i#*=}
       ;;
+      --tests-filter=*)
+        yetus_comma_to_array VOTE_FILTER "${i#*=}"
+      ;;
       --tpglobaltimer=*)
         GLOBALTIMER=${i#*=}
       ;;
@@ -892,7 +927,8 @@ function parse_args
 
   docker_parse_args "$@"
 
-  if [[ -z "${PATCH_OR_ISSUE}" ]]; then
+  if [[ -z "${PATCH_OR_ISSUE}"
+       && "${BUILDMODE}" = patch ]]; then
     yetus_usage
     exit 1
   fi
@@ -940,6 +976,7 @@ function parse_args
     PATCH_DIR="${USER_PATCH_DIR}"
   fi
 
+  # we need absolute dir for PATCH_DIR
   cd "${STARTINGDIR}" || cleanup_and_exit 1
   if [[ ! -d ${PATCH_DIR} ]]; then
     mkdir -p "${PATCH_DIR}"
@@ -950,10 +987,20 @@ function parse_args
       cleanup_and_exit 1
     fi
   fi
-
-  # we need absolute dir for PATCH_DIR
   PATCH_DIR=$(yetus_abs "${PATCH_DIR}")
 
+  # we need absolute dir for ${CONSOLE_REPORT_FILE}
+  if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
+    touch "${CONSOLE_REPORT_FILE}"
+    if [[ $? != 0 ]]; then
+      yetus_error "ERROR: cannot write to ${CONSOLE_REPORT_FILE}. Disabling console report
file."
+      unset CONSOLE_REPORT_FILE
+    else
+      j="${CONSOLE_REPORT_FILE}"
+      CONSOLE_REPORT_FILE=$(yetus_abs "${j}")
+    fi
+  fi
+
   if [[ ${RESETREPO} == "true" ]] ; then
     yetus_add_entry EXEC_MODES ResetRepo
   fi
@@ -1000,7 +1047,7 @@ function find_buildfile_dir
       yetus_debug "ERROR: ${buildfile} is not found."
       return 1
     else
-      dir=$(dirname "${dir}")
+      dir=$(faster_dirname "${dir}")
     fi
   done
 }
@@ -1013,17 +1060,29 @@ function find_buildfile_dir
 function find_changed_files
 {
   declare line
+  declare oldifs
 
-  # get a list of all of the files that have been changed,
-  # except for /dev/null (which would be present for new files).
-  # Additionally, remove any a/ b/ patterns at the front of the patch filenames.
-  # shellcheck disable=SC2016
-  while read -r line; do
-    CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
-  done < <(
-    ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}}
-    /^diff --git /   { p($3); p($4) }
-    /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
+  case "${BUILDMODE}" in
+    full)
+      echo "Building a list of all files in the source tree"
+      oldifs=${IFS}
+      IFS=$'\n'
+      CHANGED_FILES=($(git ls-files))
+      IFS=${oldifs}
+    ;;
+    patch)
+      # get a list of all of the files that have been changed,
+      # except for /dev/null (which would be present for new files).
+      # Additionally, remove any a/ b/ patterns at the front of the patch filenames.
+      # shellcheck disable=SC2016
+      while read -r line; do
+        CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
+      done < <(
+        ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}}
+        /^diff --git /   { p($3); p($4) }
+        /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
+      ;;
+    esac
 }
 
 ## @description Check for directories to skip during
@@ -1056,7 +1115,7 @@ function module_skipdir
     if [[ ${dir} == "." || ${dir} == "/" ]]; then
       return 0
     else
-      dir=$(dirname "${dir}")
+      dir=$(faster_dirname "${dir}")
       yetus_debug "Trying to skip: ${dir}"
     fi
   done
@@ -1093,8 +1152,15 @@ function find_changed_modules
   if [[ -z ${buildfile} ]]; then
     tmpmods=(".")
   else
+
     # Now find all the modules that were changed
     for i in "${CHANGED_FILES[@]}"; do
+
+      # TODO: optimize this
+      if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then
+        continue
+      fi
+
       dirt=$(dirname "${i}")
 
       module_skipdir "${dirt}"
@@ -1263,8 +1329,10 @@ function git_checkout
 
     currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
     if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
-      echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
-      echo "WARNING: Continuing anyway..."
+      if [[ "${BUILDMODE}" = patch ]]; then
+        echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
+        echo "WARNING: Continuing anyway..."
+      fi
       PATCH_BRANCH=${currentbranch}
     fi
   fi
@@ -1389,8 +1457,11 @@ function determine_issue
 ## @replaceable  no
 function determine_needed_tests
 {
-  local i
-  local plugin
+  declare i
+  declare plugin
+
+  big_console_header "Determining needed tests"
+  echo "(Depending upon input size and number of plug-ins, this may take a while)"
 
   for i in "${CHANGED_FILES[@]}"; do
     yetus_debug "Determining needed tests for ${i}"
@@ -1562,6 +1633,7 @@ function check_reexec
   declare tpdir
   declare copy=false
   declare testdir
+  declare plugin
 
   if [[ ${REEXECED} == true ]]; then
     big_console_header "Re-exec mode detected. Continuing."
@@ -1583,7 +1655,7 @@ function check_reexec
     fi
   done
 
-  if [[ ${copy} == true ]]; then
+  if [[ ${copy} == true && "${BUILDMODE}" != full ]]; then
     big_console_header "precommit patch detected"
 
     if [[ ${RESETREPO} == false ]]; then
@@ -1622,9 +1694,11 @@ function check_reexec
     # if we are doing docker, then we re-exec, but underneath the
     # container
 
-    if declare -f ${BUILDTOOL}_docker_support >/dev/null; then
-      "${BUILDTOOL}_docker_support"
-    fi
+    for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${BUGSYSTEMS} ${TESTTYPES} ${TESTFORMATS};
do
+      if declare -f ${plugin}_docker_support >/dev/null; then
+        "${plugin}_docker_support"
+      fi
+    done
 
     TESTPATCHMODE="${USER_PARAMS[*]}"
     if [[ -n "${BUILD_URL}" ]]; then
@@ -1726,7 +1800,9 @@ function modules_messages
   declare statusjdk
   declare multijdkmode=false
 
-  if [[ ${repostatus} == branch ]]; then
+  if [[ "${BUILDMODE}" == full ]]; then
+    repo="the source"
+  elif [[ "${repostatus}" == branch ]]; then
     repo=${PATCH_BRANCH}
   else
     repo="the patch"
@@ -1843,7 +1919,9 @@ function modules_workers
   declare result=0
   declare argv
 
-  if [[ ${repostatus} == branch ]]; then
+  if [[ "${BUILDMODE}" = full ]]; then
+    repo="the source"
+  elif [[ ${repostatus} == branch ]]; then
     repo=${PATCH_BRANCH}
   else
     repo="the patch"
@@ -2214,8 +2292,6 @@ function runtests
 
     verify_patchdir_still_exists
     check_unittests
-
-    (( RESULT = RESULT + $? ))
   fi
 
   for plugin in ${TESTTYPES}; do
@@ -2225,7 +2301,6 @@ function runtests
       yetus_debug "Running ${plugin}_tests"
       #shellcheck disable=SC2086
       ${plugin}_tests
-      (( RESULT = RESULT + $? ))
     fi
   done
 }
@@ -2368,12 +2443,16 @@ function generic_calcdiff_status
   ((samepatch=numpatch-addpatch))
   ((fixedpatch=numbranch-numpatch+addpatch))
 
-  printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \
-    "${addpatch}" \
-    "${samepatch}" \
-    "${fixedpatch}" \
-    "${numpatch}" \
-    "${numbranch}"
+  if [[ "${BUILDMODE}" = full ]]; then
+    printf "has %i issues." "${addpatch}"
+  else
+    printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \
+      "${addpatch}" \
+      "${samepatch}" \
+      "${fixedpatch}" \
+      "${numpatch}" \
+      "${numbranch}"
+  fi
 }
 
 ## @description  Helper routine for plugins to ask projects, etc
@@ -2581,7 +2660,7 @@ function generic_post_handler
     return 0
   fi
 
-  big_console_header "Patch ${testtype} verification"
+  big_console_header "${testtype} verification: ${BUILDMODE}"
 
   for jdkindex in ${JDK_DIR_LIST}; do
     if [[ ${multijdkmode} == true ]]; then
@@ -2705,9 +2784,9 @@ function compile
   fi
 
   if [[ ${codebase} = "branch" ]]; then
-    big_console_header "Pre-patch ${PATCH_BRANCH} compilation"
+    big_console_header "${PATCH_BRANCH} compilation: pre-patch"
   else
-    big_console_header "Patch compilation"
+    big_console_header "${PATCH_BRANCH} compilation: ${BUILDMODE}"
   fi
 
   yetus_debug "Is JVM Required? ${JVM_REQUIRED}"
@@ -2880,26 +2959,33 @@ function initialize
 
   echo "Modes: ${EXEC_MODES}"
 
-  locate_patch
+  if [[ "${BUILDMODE}" = patch ]]; then
+    locate_patch
 
-  # from here on out, we'll be in ${BASEDIR} for cwd
-  # plugins need to pushd/popd if they change.
-  git_checkout
+    # from here on out, we'll be in ${BASEDIR} for cwd
+    # plugins need to pushd/popd if they change.
+    git_checkout
+
+    determine_issue
+    if [[ "${ISSUE}" == 'Unknown' ]]; then
+      echo "Testing patch on ${PATCH_BRANCH}."
+    else
+      echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
+    fi
+
+    patchfile_dryrun_driver "${PATCH_DIR}/patch"
+    if [[ $? != 0 ]]; then
+      ((RESULT = RESULT + 1))
+      yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}."
+      add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase
required? Wrong Branch? See ${PATCH_NAMING_RULE} for help."
+      bugsystem_finalreport 1
+      cleanup_and_exit 1
+    fi
 
-  determine_issue
-  if [[ "${ISSUE}" == 'Unknown' ]]; then
-    echo "Testing patch on ${PATCH_BRANCH}."
   else
-    echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
-  fi
 
-  patchfile_dryrun_driver "${PATCH_DIR}/patch"
-  if [[ $? != 0 ]]; then
-    ((RESULT = RESULT + 1))
-    yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}."
-    add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase
required? Wrong Branch? See ${PATCH_NAMING_RULE} for help."
-    bugsystem_finalreport 1
-    cleanup_and_exit 1
+    git_checkout
+
   fi
 
   find_changed_files
@@ -2966,27 +3052,29 @@ function import_core
 
 import_core
 
-initialize "$@"
+if [[ "${BINNAME}" =~ qbt ]]; then
+  initialize --empty-patch "$@"
+else
+  initialize "$@"
+fi
 
 prechecks
 
-patchfiletests
-((RESULT=RESULT+$?))
+if [[ "${BUILDMODE}" = patch ]]; then
+  patchfiletests
 
-compile_cycle branch
-((RESULT=RESULT+$?))
+  compile_cycle branch
 
-distclean
+  distclean
 
-apply_patch_file
+  apply_patch_file
 
-compute_gitdiff
+  compute_gitdiff
+fi
 
 compile_cycle patch
-((RESULT=RESULT+$?))
 
 runtests
-((RESULT=RESULT+$?))
 
 finish_vote_table
 


Mime
View raw message