Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@yetus.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: aw@apache.org
To: commits@yetus.apache.org
Message-Id: <15f2d585c4d24cfe853b701d880057da@git.apache.org>
Subject: yetus git commit: YETUS-293. archiving functionality
Date: Mon, 20 Jun 2016 16:38:13 +0000 (UTC)
archived-at: Mon, 20 Jun 2016 16:38:15 -0000

Repository: yetus
Updated Branches:
  refs/heads/master b3f28a4d9 -> e4e4903e4


YETUS-293. archiving functionality

Signed-off-by: Sean Busbey <busbey@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e4e4903e
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e4e4903e
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e4e4903e

Branch: refs/heads/master
Commit: e4e4903e40b730b5bf28e55a44a0251bdc14f518
Parents: b3f28a4
Author: Allen Wittenauer <aw@apache.org>
Authored: Thu Jun 2 09:13:19 2016 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Mon Jun 20 09:37:54 2016 -0700

----------------------------------------------------------------------
 .../documentation/in-progress/precommit-qbt.md  | 15 +++++
 precommit/core.d/01-common.sh                   |  5 ++
 precommit/test-patch-docker/Dockerfile          |  3 +-
 precommit/test-patch.sh                         | 58 ++++++++++++++++++++
 4 files changed, 80 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/e4e4903e/asf-site-src/source/documentation/in-progress/precommit-qbt.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-qbt.md b/asf-site-src/source/documentation/in-progress/precommit-qbt.md
index badaf60..91d1749 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-qbt.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-qbt.md
@@ -27,6 +27,8 @@ It is meant to be a way to easily get test-patch's output on your
 current source tree.  It is suitable to be run as a regularly
 scheduled build as part of your overall development strategy.
 
+# Reporting
+
 When using an automation tool, it may be useful to use the
 `--console-report-file` option to send the summary email to a
 file. This can then be used with systems like Jenkin's
@@ -67,3 +69,16 @@ suitable for email.  It contains just the barebones information needed to get
 information on failures: what voted -1, what tests failed, what subsystems are long
 running (configurable with the `--brief-report-long` opton), and a list of any
 attached log files.
+
+NOTE: Be aware that ASF mailing lists do not allow HTML formatted email.
+
+# Archiving
+
+It may be useful to save off certain files while qbt is running for more
+post-processing by another utility.  If the `rsync` command is available,
+then the archiving functionality may be used.
+
+The `--archive-list` option takes a comma separated list of `find -name`
+patterns and copies them to the patch directory's archiver subdirectory.
+It will preserve the directory structure of the source tree so that
+multiple matching file names will be preserved.

http://git-wip-us.apache.org/repos/asf/yetus/blob/e4e4903e/precommit/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh
index 3471dc3..6697cc2 100755
--- a/precommit/core.d/01-common.sh
+++ b/precommit/core.d/01-common.sh
@@ -87,6 +87,8 @@ function common_defaults
       SED=${SED:-sed}
     ;;
   esac
+
+  RSYNC=${RSYNC:-rsync}
 }
 
 ## @description  Interpret the common command line parameters used by test-patch,
@@ -163,6 +165,9 @@ function common_args
       --project=*)
         PROJECT_NAME=${i#*=}
       ;;
+      --rsync-cmd=*)
+        RSYNC=${i#*=}
+      ;;
       --skip-system-plugins)
         LOAD_SYSTEM_PLUGINS=false
       ;;

http://git-wip-us.apache.org/repos/asf/yetus/blob/e4e4903e/precommit/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile
index 5c007cd..5c756cf 100644
--- a/precommit/test-patch-docker/Dockerfile
+++ b/precommit/test-patch-docker/Dockerfile
@@ -46,7 +46,8 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
     pkg-config \
     python \
     python2.7 \
-    python-pip
+    python-pip \
+    rsync
 
 # Fixing the Apache commons / Maven dependency problem under Ubuntu:
 # See http://wiki.apache.org/commons/VfsProblems

http://git-wip-us.apache.org/repos/asf/yetus/blob/e4e4903e/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index a818af0..d7dd3d7 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -681,6 +681,7 @@ function yetus_usage
   echo ""
   echo "Options:"
   echo ""
+  yetus_add_option "--archive-list=<list>" "Comma delimited list of pattern matching notations to copy to patch-dir"
   yetus_add_option "--basedir=<dir>" "The directory to apply the patch to (default current directory)"
   yetus_add_option "--branch=<ref>" "Forcibly set the branch"
   yetus_add_option "--branch-default=<ref>" "If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')"
@@ -780,6 +781,10 @@ function parse_args
 
   for i in "$@"; do
     case ${i} in
+      --archive-list=*)
+        yetus_comma_to_array ARCHIVE_LIST "${i#*=}"
+        yetus_debug "Set to archive: ${ARCHIVE_LIST[*]}"
+      ;;
       --bugcomments=*)
         BUGCOMMENTS=${i#*=}
         BUGCOMMENTS=${BUGCOMMENTS//,/ }
@@ -1728,6 +1733,53 @@ function check_reexec
   fi
 }
 
+## @description  Save file names and directory to the patch dir
+## @audience     public
+## @stability    evolving
+## @replaceable  no
+function archive
+{
+  declare pmn
+  declare fn
+  declare line
+  declare srcdir
+  declare tmpfile="${PATCH_DIR}/tmp.$$.${RANDOM}"
+
+  if [[ ${#ARCHIVE_LIST[@]} -eq 0 ]]; then
+    return
+  fi
+
+  if ! verify_command "rsync" "${RSYNC}"; then
+    yetus_error "WARNING: Cannot use the archive function"
+    return
+  fi
+
+  yetus_debug "Starting archiving process"
+  # get the list of files. these will be with
+  # the full path
+  # (this is pretty expensive)
+
+  rm "${tmpfile}" 2>/dev/null
+  for pmn in "${ARCHIVE_LIST[@]}"; do
+    find "${BASEDIR}" -name "${pmn}" >> "${tmpfile}"
+  done
+
+  # read the list, stripping of both
+  # the BASEDIR and any leading /.
+  # with our filename fragment,
+  # call faster_dirname with a prepended /
+  while read -r line; do
+    yetus_debug "Archiving: ${line}"
+    srcdir=$(faster_dirname "/${line}")
+    mkdir -p "${PATCH_DIR}/archiver${srcdir}"
+    "${RSYNC}" -av "${BASEDIR}/${line}" "${PATCH_DIR}/archiver${srcdir}" >/dev/null 2>&1
+  done < <("${SED}" -e "s,${BASEDIR},,g" \
+      -e "s,^/,,g" "${tmpfile}")
+  rm "${tmpfile}" 2>/dev/null
+  yetus_debug "Ending archiving process"
+
+}
+
 ## @description  Reset the test results
 ## @audience     public
 ## @stability    evolving
@@ -2294,6 +2346,7 @@ function runtests
       ${plugin}_tests
     fi
   done
+  archive
 }
 
 ## @description  Calculate the differences between the specified files
@@ -2749,6 +2802,7 @@ function compile_nonjvm
       yetus_debug "Running ${plugin}_compile ${codebase} ${multijdkmode}"
       "${plugin}_compile" "${codebase}" "${multijdkmode}"
       ((result = result + $?))
+      archive
     fi
   done
 
@@ -2812,6 +2866,7 @@ function compile_cycle
       if [[ $? -gt 0 ]]; then
         ((result = result+1))
       fi
+      archive
     fi
   done
 
@@ -2825,6 +2880,7 @@ function compile_cycle
       if [[ $? -gt 0 ]]; then
         ((result = result+1))
       fi
+      archive
     fi
   done
 
@@ -2836,6 +2892,7 @@ function compile_cycle
       if [[ $? -gt 0 ]]; then
         ((result = result+1))
       fi
+      archive
     fi
   done
 
@@ -2866,6 +2923,7 @@ function patchfiletests
       if [[ $? -gt 0 ]]; then
         ((result = result+1))
       fi
+      archive
     fi
   done