From commits-return-1166-archive-asf-public=cust-asf.ponee.io@yetus.apache.org Sun Nov 11 23:19:22 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 3D8401807A5 for ; Sun, 11 Nov 2018 23:19:19 +0100 (CET) Received: (qmail 7670 invoked by uid 500); 11 Nov 2018 22:19:18 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 7395 invoked by uid 99); 11 Nov 2018 22:19:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 11 Nov 2018 22:19:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4C6C8E125C; Sun, 11 Nov 2018 22:19:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: commits@yetus.apache.org Date: Sun, 11 Nov 2018 22:19:27 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [11/17] yetus git commit: YETUS-15. build environment http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/smart-apply-patch.sh b/precommit/src/main/shell/smart-apply-patch.sh new file mode 100755 index 0000000..a5dd09b --- /dev/null +++ b/precommit/src/main/shell/smart-apply-patch.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +#shellcheck disable=SC2034 +QATESTMODE=false + +# dummy functions +function add_vote_table +{ + true +} + +function add_footer_table +{ + true +} + +function big_console_header +{ + true +} + +function add_test +{ + true +} + +## @description Clean the filesystem as appropriate and then exit +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function cleanup_and_exit +{ + local result=$1 + + if [[ ${PATCH_DIR} =~ ^/tmp/yetus + && -d ${PATCH_DIR} ]]; then + rm -rf "${PATCH_DIR}" + fi + + # shellcheck disable=SC2086 + exit ${result} +} + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + common_defaults +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function yetus_usage +{ + echo "smart-apply-patch.sh [OPTIONS] patch" + echo "" + echo "Where:" + echo " patch is a file, URL, or bugsystem-compatible location of the patch file" + echo "" + echo "Options:" + echo "" + yetus_add_option "--committer" "Apply patches like a boss." + yetus_add_option "--debug" "If set, then output some extra stuff to stderr" + yetus_add_option "--dry-run" "Check for patch viability without applying" + yetus_add_option "--list-plugins" "List all installed plug-ins and then exit" + yetus_add_option "--modulelist=" "Specify additional modules to test (comma delimited)" + yetus_add_option "--offline" "Avoid connecting to the Internet" + yetus_add_option "--patch-dir=" "The directory for working and output files (default '/tmp/yetus-(random))" + yetus_add_option "--personality=" "The personality file to load" + yetus_add_option "--plugins=" "Specify which plug-ins to add/delete (comma delimited; use 'all' for all found)" + yetus_add_option "--project=" "The short name for project currently using test-patch (default 'yetus')" + yetus_add_option "--skip-system-plugins" "Do not load plugins from ${BINDIR}/test-patch.d" + yetus_add_option "--user-plugins=" "A directory of user provided plugins. see test-patch.d for examples (default empty)" + yetus_add_option "--version" "Print release version information and exit" + yetus_add_option "--gpg-sign" "GPG sign the commit using gpg keys" + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage + + echo "" + echo "Shell binary overrides:" + yetus_add_option "--awk-cmd=" "The 'awk' command to use (default 'awk')" + yetus_add_option "--curl-cmd=" "The 'curl' command to use (default 'curl')" + yetus_add_option "--diff-cmd=" "The GNU-compatible 'diff' command to use (default 'diff')" + yetus_add_option "--file-cmd=" "The 'file' command to use (default 'file')" + yetus_add_option "--git-cmd=" "The 'git' command to use (default 'git')" + yetus_add_option "--grep-cmd=" "The 'grep' command to use (default 'grep')" + yetus_add_option "--patch-cmd=" "The 'patch' command to use (default 'patch')" + yetus_add_option "--sed-cmd=" "The 'sed' command to use (default 'sed')" + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage + + echo "" + importplugins + + unset TESTFORMATS + unset TESTTYPES + unset BUILDTOOLS + + for plugin in ${BUGSYSTEMS}; do + if declare -f ${plugin}_usage >/dev/null 2>&1; then + echo "" + echo "${plugin} plugin usage options:" + "${plugin}_usage" + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage + fi + done +} + +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @param $@ +## @return May exit on failure +function parse_args +{ + local i + + common_args "$@" + + for i in "$@"; do + case ${i} in + --committer) + COMMITMODE=true + ;; + --gpg-sign) + GPGSIGN=true + ;; + --dry-run) + PATCH_DRYRUNMODE=true + ;; + --*) + ## PATCH_OR_ISSUE can't be a --. So this is probably + ## a plugin thing. + continue + ;; + *) + PATCH_OR_ISSUE=${i#*=} + ;; + esac + done + + if [[ ! -d ${PATCH_DIR} ]]; then + mkdir -p "${PATCH_DIR}" + if [[ $? != 0 ]] ; then + yetus_error "ERROR: Unable to create ${PATCH_DIR}" + cleanup_and_exit 1 + fi + fi +} + +## @description git am dryrun +## @replaceable no +## @audience private +## @stability evolving +function gitam_dryrun +{ + + # there is no dryrun method for git-am, so just + # use apply instead. + gitapply_dryrun "$@" + + if [[ ${PATCH_METHOD} = "gitapply" ]]; then + PATCH_METHOD="gitam" + fi +} + +## @description git am signoff +## @replaceable no +## @audience private +## @stability evolving +function gitam_apply +{ + declare patchfile=$1 + declare gpg=$2 + + if [[ ${gpg} = true ]]; then + EXTRA_ARGS="-S" + fi + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am.log" \ + "${GIT}" am --signoff ${EXTRA_ARGS} --whitespace=fix "-p${PATCH_LEVEL}" "${patchfile}" + RESULT=$? + ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-am.log" + + # fallback + if [[ ${RESULT} -gt 0 && ${PATCH_SYSTEM} == 'jira' ]]; then + echo "Use git apply and commit with the information from jira." + gitapply_and_commit "${patchfile}" + fi +} + +## @description get author and summary from jira and commit it. +## if the author and the summary contains " or *, +## the function does not work correctly because +## the characters are used for delimiters. +## @replaceable no +## @audience private +## @stability evolving +function gitapply_and_commit +{ + declare patchfile=$1 + declare jsontmpfile + declare assigneeline + declare assigneefile + declare name + declare email + declare author + declare summary + + yetus_debug "gitapply_and_commit: fetching ${JIRA_URL}/rest/api/2/issue/${PATCH_OR_ISSUE}" + if ! jira_http_fetch "rest/api/2/issue/${PATCH_OR_ISSUE}" "${PATCH_DIR}/issue"; then + yetus_debug "gitapply_and_commit: not a JIRA." + return 1 + fi + + jsontmpfile="${PATCH_DIR}/jsontmpfile" + # cannot set " as delimiter for cut command in script, so replace " with * + tr ',' '\n' < "${PATCH_DIR}/issue" | ${SED} 's/\"/*/g' > "${jsontmpfile}" + + assigneeline=$(${GREP} -n -E "^\*assignee\*:" "${jsontmpfile}" | cut -f1 -d":") + assigneefile="${PATCH_DIR}/assigneefile" + tail -n +"${assigneeline}" "${jsontmpfile}" | head -n 20 > "${assigneefile}" + + name=$(${GREP} "displayName" "${assigneefile}" | cut -f4 -d"*") + email=$(${GREP} "emailAddress" "${assigneefile}" | cut -f4 -d"*" \ + | ${SED} 's/ at /@/g' | ${SED} 's/ dot /./g') + author="${name} <${email}>" + summary=$(${GREP} -E "^\*summary\*:" "${jsontmpfile}" | cut -f4 -d"*") + gitapply_apply "${patchfile}" + ${GIT} add --all + echo "Committing with author: ${author}, summary: ${summary}" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am-fallback.log" \ + "${GIT}" commit ${EXTRA_ARGS} --signoff -m "${PATCH_OR_ISSUE}. ${summary}" \ + --author="${author}" +} + +## @description import core library routines +## @audience private +## @stability evolving +function import_core +{ + declare filename + + for filename in "${BINDIR}/core.d"/*; do + # shellcheck disable=SC1091 + # shellcheck source=core.d/01-common.sh + . "${filename}" + done +} + +trap "cleanup_and_exit 1" HUP INT QUIT TERM + +import_core + +setup_defaults + +parse_args "$@" + +importplugins +yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" +unset BUILDTOOLS +unset TESTTYPES +unset TESTFORMATS + +parse_args_plugins "$@" + +plugins_initialize + +locate_patch + +if [[ ${COMMITMODE} = true ]]; then + status=$(${GIT} status --porcelain) + if [[ "$status" != "" ]] ; then + yetus_error "ERROR: Can't use --committer option in a workspace that contains the following modifications:" + yetus_error "${status}" + cleanup_and_exit 1 + fi + PATCH_METHODS=("gitam" "${PATCH_METHODS[@]}") +fi + +patchfile_dryrun_driver "${PATCH_DIR}/patch" +RESULT=$? + +if [[ ${RESULT} -gt 0 ]]; then + yetus_error "ERROR: Aborting! ${PATCH_OR_ISSUE} cannot be verified." + cleanup_and_exit ${RESULT} +fi + +if [[ ${PATCH_DRYRUNMODE} == false ]]; then + patchfile_apply_driver "${PATCH_DIR}/patch" "${GPGSIGN}" + RESULT=$? +fi + +if [[ ${COMMITMODE} = true + && ${PATCH_METHOD} != "gitam" ]]; then + yetus_debug "Running git add -A" + git add -A +fi + +cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile new file mode 100644 index 0000000..31f9f3b --- /dev/null +++ b/precommit/src/main/shell/test-patch-docker/Dockerfile @@ -0,0 +1,201 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +############### +# +# Example Apache Yetus Dockerfile that includes all functionality supported +# as well as enough bits to build and release Apache Yetus itself. +# +############### + +FROM ubuntu:xenial + +## NOTE to committers: if this gets moved from Xenial to something else, be +## sure to also fix the gpg link in asf-site-src as appropriate + +WORKDIR /root +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Install some basic Apache Yetus requirements +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ + curl \ + git \ + locales \ + pkg-config \ + rsync \ + software-properties-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +### +# Set the locale +### +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +#### +# Install java (first, since we want to dicate what form of Java) +#### + +#### +# OpenJDK 8 +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +####### +# OpenJDK 9 +# w/workaround for +# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191 +####### +RUN apt-get -q update && apt-get -o Dpkg::Options::="--force-overwrite" \ + -q install --no-install-recommends -y \ + openjdk-9-jdk-headless \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +####### +# Set default Java +####### +# +# By default, OpenJDK sets the default Java to the highest version. +# We want the opposite, soooooo.... +# +RUN update-java-alternatives --set java-1.8.0-openjdk-amd64 +RUN update-alternatives --get-selections | grep -i jdk | \ + while read -r line; do \ + alternative=$(echo "$line" | awk '{print $1}'); \ + path=$(echo "$line" | awk '{print $3}'); \ + newpath=${path//java-9/java-8}; \ + update-alternatives --set "$alternative" "$newpath"; \ + done + + +#### +# Install ant +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#### +# Install GNU automake, GNU make, and related +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y autoconf automake libtool make \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#### +# Install bats (TAP-capable unit testing for shell scripts) +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y bats \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#### +# Install cmake +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +###### +# Install findbugs +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +ENV FINDBUGS_HOME /usr + +#### +# Install GNU C/C++ +#### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y g++ gcc libc-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +###### +# Install maven +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y maven \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +###### +# Install perl +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y perl libperl-critic-perl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +###### +# Install python and pylint +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \ + python2.7 \ + python-pip \ + python-pkg-resources \ + python-setuptools \ + python-wheel \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN pip2 install -v pylint==1.9.2 python-dateutil==2.7.3 + +#### +# Install ruby, rubocop, and ruby-lint +### +RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc +RUN apt-get -q update && apt-get -q install --no-install-recommends -y ruby ruby-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN gem install rake +RUN gem install rubocop +RUN gem install ruby-lint + +#### +# Install shellcheck (shell script lint) +#### +RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 +RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +### +# Install hadolint +#### +RUN curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && \ + chmod a+rx /bin/hadolint && \ + shasum -a 512 /bin/hadolint | \ + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +#### +# YETUS CUT HERE +# Anthing after the above line is ignored by Yetus, so could +# include other requirements not needed by your development +# (but not build) environment +### http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch-docker/Dockerfile-endstub ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile-endstub b/precommit/src/main/shell/test-patch-docker/Dockerfile-endstub new file mode 100644 index 0000000..ce76491 --- /dev/null +++ b/precommit/src/main/shell/test-patch-docker/Dockerfile-endstub @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ADD launch-test-patch.sh /testptch/launch-test-patch.sh +RUN mkdir /testptch/extras +RUN chmod a+rwx /testptch/extras +RUN chmod a+rx /testptch/launch-test-patch.sh +CMD /testptch/launch-test-patch.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch-docker/launch-test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch-docker/launch-test-patch.sh b/precommit/src/main/shell/test-patch-docker/launch-test-patch.sh new file mode 100755 index 0000000..b170cd2 --- /dev/null +++ b/precommit/src/main/shell/test-patch-docker/launch-test-patch.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -a OVERWRITEARGS + +OVERWRITEARGS=("--reexec") +OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--dockermode") +OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--basedir=${BASEDIR}") + +cd "${BASEDIR}" || exit 1 + +if [[ -n ${JAVA_HOME} + && ! -d ${JAVA_HOME} ]]; then + echo "JAVA_HOME: ${JAVA_HOME} does not exist. Dockermode: attempting to switch to another." 1>&2 + JAVA_HOME="" +fi + +if [[ -z ${JAVA_HOME} ]]; then + JAVA_HOME=$(find /usr/lib/jvm/ -name "java-*" -type d | tail -1) + export JAVA_HOME + if [[ -n "${JAVA_HOME}" ]]; then + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--java-home=${JAVA_HOME}") + echo "Setting ${JAVA_HOME} as the JAVA_HOME." + fi +fi + +# Avoid out of memory errors in builds +MAVEN_OPTS=${MAVEN_OPTS:-"-Xms256m -Xmx1g"} +export MAVEN_OPTS + +# strip out --docker param to prevent re-exec again +TESTPATCHMODE=${TESTPATCHMODE/--docker } +TESTPATCHMODE=${TESTPATCHMODE%--docker} + +PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) +OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--patch-dir=${PATCH_DIR}") +OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--user-plugins=${PATCH_DIR}/precommit/user-plugins") +if [[ -f "${PATCH_DIR}/precommit/unit_test_filter_file.txt" ]]; then + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--unit-test-filter-file=${PATCH_DIR}/precommit/unit_test_filter_file.txt") +fi + +# if patch system is generic, then it's either a local +# patch file or was in some other way not pulled from a bug +# system. So we need to rescue it and then tell +# test-patch where to find it. +if [[ "${PATCH_SYSTEM}" = generic ]]; then + cp -p "${PATCH_DIR}/patch" /testptch/extras/patch + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "/testptch/extras/patch") +fi + +if [[ -f /testptch/console.txt ]]; then + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--console-report-file=/testptch/console.txt") +fi + +if [[ -f /testptch/brief.txt ]]; then + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--brief-report-file=/testptch/brief.txt") +fi + +if [[ -f /testptch/report.htm ]]; then + OVERWRITEARGS=("${OVERWRITEARGS[@]}" "--html-report-file=/testptch/report.htm") +fi + +cd "${PATCH_DIR}/precommit/" || exit 1 +#shellcheck disable=SC2086 +"${PATCH_DIR}/precommit/test-patch.sh" \ + ${TESTPATCHMODE} \ + "${OVERWRITEARGS[@]}" http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/ant.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/ant.sh b/precommit/src/main/shell/test-patch.d/ant.sh new file mode 100755 index 0000000..4c81741 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/ant.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# there's nothing in here public, so don't publish docs +# SHELLDOC-IGNORE + +if [[ -z "${ANT_HOME:-}" ]]; then + ANT=ant +else + ANT=${ANT_HOME}/bin/ant +fi + +add_build_tool ant + +declare -a ANT_ARGS=("-noinput") + +function ant_usage +{ + yetus_add_option "--ant-cmd=" "The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')" +} + +function ant_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --ant-cmd=*) + ANT=${i#*=} + ;; + esac + done + + # if we requested offline, pass that to ant + if [[ ${OFFLINE} == "true" ]]; then + ANT_ARGS=("${ANT_ARGS[@]}" -Doffline=) + fi +} + +function ant_initialize +{ + # we need to do this before docker kicks in + if [[ -e "${HOME}/.ivy2" + && ! -d "${HOME}/.ivy2" ]]; then + yetus_error "ERROR: ${HOME}/.ivy2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.ivy2" ]]; then + yetus_debug "Creating ${HOME}/.ivy2" + mkdir -p "${HOME}/.ivy2" + fi +} + +function ant_precheck +{ + declare ant_version + + if ! verify_command ant "${ANT}"; then + add_vote_table -1 ant "ERROR: ant is not available." + return 1 + fi + # finally let folks know what version they'll be dealing with. + ant_version=$(${ANT} -version 2>/dev/null) + add_footer_table ant "version: ${ant_version}" + return 0 +} + +function ant_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ build\.xml$ + || ${filename} =~ ivy\.xml$ ]]; then + yetus_debug "tests/compile: ${filename}" + add_test compile + fi +} + +function ant_buildfile +{ + echo "build.xml" +} + +function ant_executor +{ + echo "${ANT}" "${ANT_ARGS[@]}" +} + +function ant_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + findbugs) + modules_workers "${repostatus}" findbugs findbugs + ;; + compile) + modules_workers "${repostatus}" compile + ;; + distclean) + modules_workers "${repostatus}" distclean clean + ;; + javadoc) + modules_workers "${repostatus}" javadoc clean javadoc + ;; + unit) + modules_workers "${repostatus}" unit + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function ant_javac_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016 + ${GREP} "\[javac\] /" "${input}" > "${output}" +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function ant_javadoc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016 + ${GREP} "\[javadoc\] /" "${input}" > "${output}" +} + +function ant_builtin_personality_modules +{ + local repostatus=$1 + local testtype=$2 + + local module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} + +function ant_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin ant personality_file_tests" + + if [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ src/test + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml + || ${filename} =~ ivy.xml + || ${filename} =~ \.java$ + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +function ant_docker_support +{ + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS}" "-v" "${HOME}/.ivy2:/home/${USER_NAME}/.ivy2") +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/asflicense.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/asflicense.sh b/precommit/src/main/shell/test-patch.d/asflicense.sh new file mode 100755 index 0000000..593e4af --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/asflicense.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +add_test_type asflicense + +function asflicense_usage +{ + yetus_add_option "--asflicense-rat-excludes=" "path to file containing exclusion patterns" + yetus_add_option "--asflicense-rat-jar=" "path to Apache Creadur Rat jar file" +} + +function asflicense_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --asflicense-rat-excludes=*) + ASFLICENSE_RAT_EXCLUDES=${i#*=} + ;; + --asflicense-rat-jar=*) + ASFLICENSE_RAT_JAR=${i#*=} + ;; + esac + done + + case ${BUILDTOOL} in + ant|gradle|maven) + add_test asflicense + ;; + *) + if [[ -f "${ASFLICENSE_RAT_JAR}" ]]; then + add_test asflicense + fi + ;; + esac +} + +## @description Verify all files have an Apache License +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function asflicense_tests +{ + local numpatch + local btfails=true + + big_console_header "Determining number of ASF License errors" + + start_clock + + personality_modules patch asflicense + case ${BUILDTOOL} in + ant) + modules_workers patch asflicense releaseaudit + ;; + gradle) + btfails=false + modules_workers patch asflicense rat + ;; + maven) + modules_workers patch asflicense -fn apache-rat:check + btfails=false + ;; + *) + if [[ -z "${ASFLICENSE_RAT_JAR}" ]]; then + return 0 + fi + + btfails=false + asflicense_writexsl "${PATCH_DIR}/asf.xsl" + if [[ -f ${ASFLICENSE_RAT_EXCLUDES} ]]; then + echo_and_redirect "${PATCH_DIR}/patch-asflicense.txt" \ + "${JAVA_HOME}/bin/java" \ + -jar "${ASFLICENSE_RAT_JAR}" \ + -s "${PATCH_DIR}/asf.xsl" \ + -E "${ASFLICENSE_RAT_EXCLUDES}" \ + -d "${BASEDIR}" + else + echo_and_redirect "${PATCH_DIR}/patch-asflicense.txt" \ + "${JAVA_HOME}/bin/java" \ + -jar "${ASFLICENSE_RAT_JAR}" \ + -s "${PATCH_DIR}/asf.xsl" \ + "${BASEDIR}" + fi + ;; + esac + + # RAT fails the build if there are license problems. + # so let's take advantage of that a bit. + if [[ $? == 0 && ${btfails} = true ]]; then + add_vote_table 1 asflicense "${BUILDMODEMSG} does not generate ASF License warnings." + return 0 + fi + + if [[ ! -f "${PATCH_DIR}/patch-asflicense.txt" ]]; then + #shellcheck disable=SC2038 + find "${BASEDIR}" -name rat.txt \ + -o -name releaseaudit_report.txt \ + -o -name rat-report.txt \ + | xargs cat > "${PATCH_DIR}/patch-asflicense.txt" + fi + + if [[ ! -s "${PATCH_DIR}/patch-asflicense.txt" ]]; then + if [[ ${btfails} = true ]]; then + # if we're here, then build actually failed + modules_messages patch asflicense true + return 1 + else + add_vote_table 0 asflicense "ASF License check generated no output?" + return 0 + fi + fi + + numpatch=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patch-asflicense.txt") + echo "" + echo "" + echo "There appear to be ${numpatch} ASF License warnings after applying the patch." + if [[ -n ${numpatch} + && ${numpatch} -gt 0 ]] ; then + add_vote_table -1 asflicense "${BUILDMODEMSG} generated ${numpatch} ASF License warnings." + + echo "Lines that start with ????? in the ASF License "\ + "report indicate files that do not have an Apache license header:" \ + > "${PATCH_DIR}/patch-asflicense-problems.txt" + + ${GREP} '\!?????' "${PATCH_DIR}/patch-asflicense.txt" \ + >> "${PATCH_DIR}/patch-asflicense-problems.txt" + + add_footer_table asflicense "@@BASE@@/patch-asflicense-problems.txt" + return 1 + fi + add_vote_table 1 asflicense "${BUILDMODEMSG} does not generate ASF License warnings." + return 0 +} + +function asflicense_writexsl +{ +cat > "${1}" << EOF + + + + + + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + + + + ! + + + + N + A + B + + !!!!! + + + + + + + + +EOF +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/author.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/author.sh b/precommit/src/main/shell/test-patch.d/author.sh new file mode 100755 index 0000000..dccd504 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/author.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type author + +## @description author usage hook +## @audience private +## @stability evolving +## @replaceable no +function author_usage +{ + yetus_add_option "--author-ignore-list=" "list of filenames to ignore (full build mode only)" +} + +## @description author parse args hook +## @audience private +## @stability evolving +## @replaceable no +function author_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --author-ignore-list=*) + yetus_comma_to_array AUTHOR_IGNORE_LIST "${i#*=}" + ;; + esac + done +} + +## @description helper function for @author tags check +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function author_generic +{ + declare authortags + declare i + declare msg + + if [[ "${BUILDMODE}" = full ]]; then + msg="source tree" + else + msg="patch" + fi + + # shellcheck disable=SC2016 + authortags=$(wc -l "${PATCH_DIR}/author-tags.txt" | "${AWK}" '{print $1}') + echo "There appear to be ${authortags} @author tags in the ${msg}." + if [[ ${authortags} != 0 ]] ; then + add_vote_table -1 @author \ + "${BUILDMODEMSG} appears to contain ${authortags} @author tags which the" \ + " community has agreed to not allow in code contributions." + add_footer_table @author "@@BASE@@/author-tags.txt" + return 1 + fi + add_vote_table +1 @author "${BUILDMODEMSG} does not contain any @author tags." + return 0 +} + +## @description Check the current patchfile for @author tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +## @param patchfile +function author_patchfile +{ + declare patchfile=$1 + # shellcheck disable=SC2155 + declare -r appname=$(basename "${BASH_SOURCE-$0}") + declare i + + if [[ "${BUILDMODE}" != patch ]]; then + return + fi + + big_console_header "Checking for @author tags: ${BUILDMODE}" + + start_clock + + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ ${appname} ]]; then + echo "Skipping @author checks as ${appname} has been patched." + add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched." + return 0 + fi + done + + ${GREP} -i -n '^[^-].*@author' "${patchfile}" >> "${PATCH_DIR}/author-tags.txt" + author_generic +} + + +## @description Check the current directory for @author tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function author_postcompile +{ + # shellcheck disable=SC2155 + declare -r appname=$(basename "${BASH_SOURCE-$0}") + declare fn + + if [[ "${BUILDMODE}" != full ]]; then + return + fi + + big_console_header "Checking for @author tags: ${BUILDMODE}" + + start_clock + + "${GIT}" grep -n -I --extended-regexp -i -e '^[^-].*@author' \ + | ${GREP} -v "${appname}" \ + >> "${PATCH_DIR}/author-tags-git.txt" + + if [[ -z "${AUTHOR_IGNORE_LIST[0]}" ]]; then + cp -p "${PATCH_DIR}/author-tags-git.txt" "${PATCH_DIR}/author-tags.txt" + else + cp -p "${PATCH_DIR}/author-tags-git.txt" "${PATCH_DIR}/author-tags.1" + for fn in "${AUTHOR_IGNORE_LIST[@]}"; do + ${GREP} -v -E "^${fn}" "${PATCH_DIR}/author-tags.1" >> "${PATCH_DIR}/author-tags.2" + mv "${PATCH_DIR}/author-tags.2" "${PATCH_DIR}/author-tags.1" + done + mv "${PATCH_DIR}/author-tags.1" "${PATCH_DIR}/author-tags.txt" + fi + + author_generic +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/autoconf.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/autoconf.sh b/precommit/src/main/shell/test-patch.d/autoconf.sh new file mode 100755 index 0000000..b1e2e2b --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/autoconf.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool autoconf + +## @description autoconf usage hook +## @audience private +## @stability evolving +## @replaceable no +function autoconf_usage +{ + yetus_add_option "--autoconf-configure-flags=" "Extra, non-'--prefix' 'configure' flags to use" +} + +## @description autoconf argument parser +## @audience private +## @stability evolving +## @param args +function autoconf_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --autoconf-configure-flags=*) + AUTOCONF_CONF_FLAGS=${i#*=} + ;; + esac + done +} + +## @description initialize autoconf +## @audience private +## @stability evolving +## @replaceable no +function autoconf_initialize +{ + if ! declare -f make_executor > /dev/null; then + yetus_error "ERROR: autoconf requires make to be enabled." + return 1 + fi +} + +## @description precheck autoconf +## @audience private +## @stability evolving +## @replaceable no +function autoconf_precheck +{ + declare autoconf_version + if ! verify_command autoconf autoreconf; then + add_vote_table -1 autoconf "autoreconf was not available." + return 1 + fi + + # finally let folks know what version they'll be dealing with. + autoconf_version=$(autoconf --version 2>/dev/null | head -n 1 2>/dev/null) + add_footer_table autoconf "version: ${autoconf_version}" + + make_precheck +} + +## @description get the name of the autoconf build filename +## @audience private +## @stability evolving +## @replaceable no +## @return autoconf build file +function autoconf_buildfile +{ + echo "Makefile.am" +} + +## @description get the name of the autoconf binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param params +function autoconf_executor +{ + make_executor "$@" +} + +## @description precompile for autoconf +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +function autoconf_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != autoconf ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "autoconf verification: ${PATCH_BRANCH}" + else + big_console_header "autoconf verification: ${BUILDMODE}" + fi + + personality_modules "${repostatus}" autoreconf + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/${repostatus}-autoconf-autoreconf" autoreconf -fi + result=$? + popd >/dev/null + + if [[ ${result} != 0 ]]; then + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table -1 autoreconf "${PATCH_BRANCH} unable to autoreconf" + else + add_vote_table -1 autoreconf "${BUILDMODEMSG} is unable to autoreconf" + fi + add_footer_table "autoreconf" "@@BASE@@/${repostatus}-autoconf-autoreconf" + return 1 + else + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table +1 autoreconf "${PATCH_BRANCH} autoreconf successful" + else + add_vote_table +1 autoreconf "${BUILDMODEMSG} can autoreconf" + fi + fi + + personality_modules "${repostatus}" configure + + pushd "${BASEDIR}" >/dev/null + #shellcheck disable=SC2086 + echo_and_redirect \ + "${PATCH_DIR}/${repostatus}-autoconf-configure" \ + ./configure \ + --prefix="${PATCH_DIR}/${repostatus}-install-dir" \ + ${AUTOCONF_CONF_FLAGS} + result=$? + popd >/dev/null + + if [[ ${result} != 0 ]]; then + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table -1 configure "${PATCH_BRANCH} unable to configure" + else + add_vote_table -1 configure "${BUILDMODEMSG} is unable to configure" + fi + add_footer_table "configure" "@@BASE@@/${repostatus}-autoconf-configure" + return 1 + else + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table +1 configure "${PATCH_BRANCH} configure successful" + else + add_vote_table +1 configure "${BUILDMODEMSG} can configure" + fi + fi + return 0 +} + +## @description autoconf worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function autoconf_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + if [[ "${tst}" = distclean ]]; then + modules_workers "${repostatus}" distclean distclean + else + make_modules_worker "$@" + fi +} + +## @description autoconf module queuer +## @audience private +## @stability evolving +## @replaceable no +function autoconf_builtin_personality_modules +{ + make_builtin_personality_modules "$@" +} + +## @description autoconf test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function autoconf_builtin_personality_file_tests +{ + declare filename=$1 + + if [[ ${filename} =~ \.m4$ + || ${filename} =~ \.in$ ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + else + make_builtin_personality_file_tests "${filename}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/briefreport.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/briefreport.sh b/precommit/src/main/shell/test-patch.d/briefreport.sh new file mode 100755 index 0000000..e536b97 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/briefreport.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# there are no public APIs here +# SHELLDOC-IGNORE + +add_bugsystem briefreport + +BRIEFOUT_LONGRUNNING=3600 + +## @description Usage info for briefreport plugin +## @audience private +## @stability evolving +## @replaceable no +function briefreport_usage +{ + yetus_add_option "--brief-report-file=" "Save a very brief, plain text report to a file" + yetus_add_option "--brief-report-long=" "Time in seconds to use as long running subsystem threshold (Default: ${BRIEFOUT_LONGRUNNING})" + +} + +## @description Option parsing for briefreport plugin +## @audience private +## @stability evolving +## @replaceable no +function briefreport_parse_args +{ + declare i + declare fn + + for i in "$@"; do + case ${i} in + --brief-report-file=*) + fn=${i#*=} + ;; + --brief-report-long=*) + BRIEFOUT_LONGRUNNING=${i#*=} + ;; + esac + done + + if [[ -n "${fn}" ]]; then + if : > "${fn}"; then + BRIEFOUT_REPORTFILE_ORIG="${fn}" + BRIEFOUT_REPORTFILE=$(yetus_abs "${BRIEFOUT_REPORTFILE_ORIG}") + else + yetus_error "WARNING: cannot create brief text report file ${fn}. Ignoring." + fi + fi +} + +## @description Give access to the brief text report file in docker mode +## @audience private +## @stability evolving +## @replaceable no +function briefreport_docker_support +{ + if [[ -n ${BRIEFOUT_REPORTFILE} ]]; then + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${BRIEFOUT_REPORTFILE}:/testptch/brief.txt") + fi +} + +## @description Only print selected information to a report file +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +## @return 0 on success +## @return 1 on failure +function briefreport_finalreport +{ + declare result=$1 + shift + declare i=0 + declare ourstring + declare vote + declare subs + declare ela + declare version + declare -a failed + declare -a long + declare -a filtered + declare hours + declare newtime + declare havelogs=false + + if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then + return + fi + + big_console_header "Writing Brief Text Report to ${BRIEFOUT_REPORTFILE}" + + + if [[ ${result} == 0 ]]; then + printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + else + printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + fi + + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + + if [[ "${vote}" = "H" ]]; then + ((i=i+1)) + continue + fi + + if [[ ${vote// } = -1 ]]; then + failed=("${failed[@]}" "${subs}") + fi + + if [[ ${vote// } = -0 ]]; then + filtered=("${filtered[@]}" "${subs}") + fi + + if [[ ${ela// } -gt ${BRIEFOUT_LONGRUNNING} ]]; then + long=("${long[@]}" "${subs}") + fi + + ((i=i+1)) + done + + tmparray=($(printf "%s\n" "${failed[@]}" | sort -u)) + failed=("${tmparray[@]}") + tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u)) + filtered=("${tmparray[@]}") + tmparray=($(printf "%s\n" "${long[@]}" | sort -u)) + long=("${tmparray[@]}") + + if [[ ${#failed[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems voted -1:" + echo " ${failed[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#filtered[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems voted -1 but" + echo "were configured to be filtered/ignored:" + echo " ${filtered[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#long[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems are considered long running:" + printf "(runtime bigger than " + # We would use clock_display here, but we don't have the + # restrictions that the vote_table has on size plus + # we're almost certainly going to be measured in hours + if [[ ${BRIEFOUT_LONGRUNNING} -ge 3600 ]]; then + hours=$((BRIEFOUT_LONGRUNNING/3600)) + newtime=$((BRIEFOUT_LONGRUNNING-hours*3600)) + printf "%sh %02sm %02ss" ${hours} $((newtime/60)) $((newtime%60)) + else + printf "%sm %02ss" $((BRIEFOUT_LONGRUNNING/60)) $((BRIEFOUT_LONGRUNNING%60)) + fi + echo ")" + echo " ${long[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + echo "" + echo "Specific tests:" + } >> "${BRIEFOUT_REPORTFILE}" + + i=0 + until [[ $i -gt ${#TP_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + { + if [[ -n "${vote// }" ]]; then + echo "" + printf " %s:\n" "${vote}" + echo "" + vote="" + fi + printf " %s\n" "${subs}" + } >> "${BRIEFOUT_REPORTFILE}" + ((i=i+1)) + done + fi + + if [[ -f "${BINDIR}/../VERSION" ]]; then + version=$(cat "${BINDIR}/../VERSION") + elif [[ -f "${BINDIR}/VERSION" ]]; then + version=$(cat "${BINDIR}/VERSION") + fi + + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then + havelogs=true + break + fi + ((i=i+1)) + done + + if [[ "${havelogs}" == true ]]; then + vote="" + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|) + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + cut -f3 -d\| | + ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") + # shellcheck disable=SC2016 + size=$(du -sh "${comment// }" | ${AWK} '{print $1}') + if [[ -n "${BUILD_URL}" ]]; then + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + cut -f3 -d\| | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + fi + { + if [[ "${subs}" != "${vote}" ]]; then + echo "" + printf " %s:\n" "${subs// }" + echo "" + vote=${subs} + fi + printf " %s [%s]\n" "${comment}" "${size}" + } >> "${BRIEFOUT_REPORTFILE}" + fi + ((i=i+1)) + done + fi + + { + echo "" + echo "Powered by" "Apache Yetus ${version} http://yetus.apache.org" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/bugzilla.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/bugzilla.sh b/precommit/src/main/shell/test-patch.d/bugzilla.sh new file mode 100755 index 0000000..9b7caeb --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/bugzilla.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +add_bugsystem bugzilla + +# personalities can override the following settings: +BUGZILLA_BASE_URL="https://bz.apache.org/bugzilla" + +function bugzilla_usage +{ + yetus_add_option "--bugzilla-base-url=" "The URL of the bugzilla server" +} + +function bugzilla_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --bugzilla-base-url=*) + BUGZILLA_BASE_URL=${i#*=} + ;; + esac + done +} + +function bugzilla_determine_issue +{ + declare input=$1 + + if [[ ! "${input}" =~ ^BZ: ]]; then + return 1 + fi + + if [[ -n "${BUGZILLA_ISSUE}" ]]; then + return 0 + fi + + # shellcheck disable=SC2016 + BUGZILLA_ISSUE=$(echo "${input}" | cut -f2 -d: ) + + # shellcheck disable=SC2034 + ISSUE=${input} + add_footer_table "Bugzilla Issue" "${BUGZILLA_ISSUE}" + return 0 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +## @return 1 on failure +function bugzilla_determine_branch +{ + return 1 +} + +function bugzilla_http_fetch +{ + declare input=$1 + declare output=$2 + + if [[ -z "${BUGZILLA_BASE_URL}" ]]; then + return 1 + fi + + ${CURL} --silent --fail \ + --output "${output}" \ + --location \ + "${BUGZILLA_BASE_URL}/${input}" +} + + +function bugzilla_locate_patch +{ + declare input=$1 + declare fileloc=$2 + declare relativeurl + + if [[ -z "${BUGZILLA_BASE_URL}" ]]; then + return 1 + fi + + bugzilla_determine_issue "${input}" + if [[ $? != 0 || -z "${BUGZILLA_ISSUE}" ]]; then + return 1 + fi + + yetus_debug "bugzilla_locate_patch: trying ${BUGZILLA_BASE_URL}/show_bug.cgi?id=${BUGZILLA_ISSUE}" + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "bugzilla_locate_patch: offline, skipping" + return 1 + fi + + bugzilla_http_fetch "show_bug.cgi?id=${BUGZILLA_ISSUE}" "${PATCH_DIR}/bugzilla" + + if [[ $? != 0 ]]; then + yetus_debug "bugzilla_locate_patch: not a Bugzilla." + return 1 + fi + + #shellcheck disable=SC2016 + relativeurl=$(${AWK} '/action=diff/ && match($0,"attachment\.cgi.id=[0-9]*"){print substr($0,RSTART,RLENGTH)}' \ + "${PATCH_DIR}/bugzilla" | \ + tail -1) + PATCHURL="${BUGZILLA_BASE_URL}${relativeurl}" + #relativeurl="${relativeurl}&action=diff&context=patch&collapsed=&headers=1&format=raw" + echo "${input} patch is being downloaded at $(date) from" + echo "${PATCHURL}" + add_footer_table "Bugzilla Patch URL" "${PATCHURL}" + bugzilla_http_fetch "${relativeurl}" "${fileloc}" + if [[ $? != 0 ]];then + yetus_error "ERROR: ${input}/${PATCHURL} could not be downloaded." + cleanup_and_exit 1 + fi + return 0 +} + +## @description Write the contents of a file to Bugzilla +## @param filename +## @stability stable +## @audience public +function bugzilla_write_comment +{ + return 0 +} + +## @description Print out the finished details to Bugzilla +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function bugzilla_finalreport +{ + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/cc.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/cc.sh b/precommit/src/main/shell/test-patch.d/cc.sh new file mode 100755 index 0000000..3241191 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/cc.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type cc + +CC_EXT_RE='(c|cc|cpp|cxx|c\+\+|h|hh|hpp|hxx|h\+\+)' + +function cc_filefilter +{ + declare filename=$1 + + shopt -s nocasematch + if [[ ${filename} =~ \.${CC_EXT_RE}$ ]]; then + shopt -u nocasematch + yetus_debug "tests/cc: ${filename}" + add_test cc + add_test compile + fi + shopt -u nocasematch +} + +## @description check for C/C++ compiler errors +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function cc_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + if ! verify_needed_test cc; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + generic_postlog_compare compile cc "${multijdkmode}" + fi +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function cc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} -i -E "^.*\.${CC_EXT_RE}\:[[:digit:]]*\:" "${input}" > "${output}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/checkstyle.sh b/precommit/src/main/shell/test-patch.d/checkstyle.sh new file mode 100755 index 0000000..5bfca02 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/checkstyle.sh @@ -0,0 +1,412 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type checkstyle + +CHECKSTYLE_TIMER=0 +CHECKSTYLE_GOAL_DEFAULT="checkstyle" +CHECKSTYLE_GOAL="${CHECKSTYLE_GOAL_DEFAULT}" +CHECKSTYLE_OPTIONS_DEFAULT="-Dcheckstyle.consoleOutput=true" +CHECKSTYLE_OPTIONS="${CHECKSTYLE_OPTIONS_DEFAULT}" + +function checkstyle_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi + fi +} + +## @description usage help for checkstyle +## @audience private +## @stability evolving +## @replaceable no +function checkstyle_usage +{ + yetus_add_option "--checkstyle-goal=" "Checkstyle maven plugin goal to use, 'check' and 'checkstyle' supported. Defaults to '${CHECKSTYLE_GOAL_DEFAULT}'." +} + +## @description parse checkstyle args +## @audience private +## @stability evolving +## @replaceable no +## @param arg +## @param .. +function checkstyle_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --checkstyle-goal=*) + CHECKSTYLE_GOAL=${i#*=} + case ${CHECKSTYLE_GOAL} in + check) + CHECKSTYLE_OPTIONS="-Dcheckstyle.consoleOutput=true -Dcheckstyle.failOnViolation=false" + ;; + checkstyle) + ;; + *) + yetus_error "Warning: checkstyle goal ${CHECKSTYLE_GOAL} not supported. It may have unexpected behavior" + ;; + esac + ;; + esac + done +} + +## @description initialize the checkstyle plug-in +## @audience private +## @stability evolving +## @replaceable no +function checkstyle_initialize +{ + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install checkstyle + fi +} + +## @description checkstyle plug-in specific difference calculator +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function checkstyle_calcdiffs +{ + declare orig=$1 + declare new=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, strip filenames:line: + # this keeps column: in an attempt to increase + # accuracy in case of multiple, repeated errors + # since the column number shouldn't change + # if the line of code hasn't been touched + # remove the numbers from the error message for comparing + # so if only the error message numbers change + # we do not report new error + # shellcheck disable=SC2016 + cut -f3- -d: "${orig}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print $1":"$2}' > "${tmp}.branch" + # shellcheck disable=SC2016 + cut -f3- -d: "${new}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print $1":"$2}' > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # removing extra marker before the chekstyle error + # message which was needed for calculations + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 | tr -d $'\x01' + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +## @description execute checkstyle +## @audience private +## @stability stable +## @replaceable no +function checkstyle_runner +{ + declare repostatus=$1 + declare tmp=${PATCH_DIR}/$$.${RANDOM} + declare j + declare i=0 + declare fn + declare savestart=${TIMER} + declare savestop + declare output + declare logfile + declare modulesuffix + declare cmd + declare logline + declare text + declare linenum + declare codeline + declare cmdresult + + # first, let's clear out any previous run information + modules_reset + + # loop through the modules we've been given + #shellcheck disable=SC2153 + until [[ $i -eq ${#MODULE[@]} ]]; do + + # start the clock per module, setup some help vars, etc + start_clock + fn=$(module_file_fragment "${MODULE[${i}]}") + modulesuffix=$(basename "${MODULE[${i}]}") + output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" + logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" + + buildtool_cwd "${i}" + + case ${BUILDTOOL} in + ant) + cmd="${ANT} \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \ + ${ANT_ARGS[*]} checkstyle" + ;; + maven) + cmd="${MAVEN} ${MAVEN_ARGS[*]} \ + checkstyle:${CHECKSTYLE_GOAL} \ + ${CHECKSTYLE_OPTIONS} \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" + ;; + *) + UNSUPPORTED_TEST=true + return 0 + ;; + esac + + # we're going to execute it and pull out + # anything that beings with a /. that's + # almost certainly checkstyle output. + # checkstyle 6.14 or upper adds severity + # to the beginning of line, so removing it + + #shellcheck disable=SC2086 + echo_and_redirect "${logfile}" ${cmd} + cmdresult=$? + + ${SED} -e 's,^\[ERROR\] ,,g' -e 's,^\[WARN\] ,,g' "${logfile}" \ + | ${GREP} ^/ \ + | ${SED} -e "s,${BASEDIR},.,g" \ + > "${tmp}" + + if [[ "${modulesuffix}" == . ]]; then + modulesuffix=root + fi + + if [[ ${cmdresult} == 0 ]]; then + module_status ${i} +1 "${logfile}" "${BUILDMODEMSG} passed checkstyle in ${modulesuffix}" + else + module_status ${i} -1 "${logfile}" "${BUILDMODEMSG} fails to run checkstyle in ${modulesuffix}" + ((result = result + 1)) + fi + + # if we have some output, we need to do more work: + if [[ -s ${tmp} && "${BUILDMODE}" = patch ]]; then + + # first, let's pull out all of the files that + # we actually care about, esp since that run + # above is likely from the entire source + # this will grealy cut down how much work we + # have to do later + + for j in "${CHANGED_FILES[@]}"; do + ${GREP} "${j}" "${tmp}" >> "${tmp}.1" + done + + # now that we have just the files we care about, + # let's unscrew it. You see... + + # checkstyle seems to do everything it possibly can + # to make it hard to process, including inconsistent + # output (sometimes it has columns, sometimes it doesn't!) + # and giving very generic errors when context would be + # helpful, esp when doing diffs. + + # in order to help calcdiff and the user out, we're + # going to reprocess the output to include the code + # line being flagged. When calcdiff gets a hold of this + # it will have the code to act as context to help + # report the correct line + + # file:linenum:(column:)error ====> + # file:linenum:code(:column)\x01:error + # \x01 will later used to identify the begining + # of the checkstyle error message + pushd "${BASEDIR}" >/dev/null + while read -r logline; do + file=$(echo "${logline}" | cut -f1 -d:) + linenum=$(echo "${logline}" | cut -f2 -d:) + text=$(echo "${logline}" | cut -f3- -d:) + codeline=$(head -n "+${linenum}" "${file}" | tail -1 ) + { + echo -n "${file}:${linenum}:${codeline}" + echo -ne "\x01" + echo ":${text}" + } >> "${output}" + done < <(cat "${tmp}.1") + + popd >/dev/null + # later on, calcdiff will turn this into code(:column):error + # compare, and then put the file:line back onto it. + else + cp -p "${tmp}" "${output}" + fi + + rm "${tmp}" "${tmp}.1" 2>/dev/null + + savestop=$(stop_clock) + #shellcheck disable=SC2034 + MODULE_STATUS_TIMER[${i}]=${savestop} + + popd >/dev/null + ((i=i+1)) + done + + TIMER=${savestart} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + checkstyle_preapply + else + checkstyle_postapply + fi +} + +function checkstyle_preapply +{ + local result + + if ! verify_needed_test checkstyle; then + return 0 + fi + + big_console_header "checkstyle: ${PATCH_BRANCH}" + + start_clock + + personality_modules branch checkstyle + checkstyle_runner branch + result=$? + modules_messages branch checkstyle true + + # keep track of how much as elapsed for us already + CHECKSTYLE_TIMER=$(stop_clock) + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postapply +{ + declare result + declare module + declare mod + declare fn + declare i=0 + declare numbranch=0 + declare numpatch=0 + declare addpatch=0 + declare summarize=true + + if ! verify_needed_test checkstyle; then + return 0 + fi + + big_console_header "checkstyle: ${BUILDMODE}" + + start_clock + + personality_modules patch checkstyle + checkstyle_runner patch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKSTYLE_TIMER}" + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + + # if there is no comparison to be done, + # we can speed this up tremendously + if [[ "${BUILDMODE}" = full ]]; then + touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" + cp -p "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/diff-checkstyle-${fn}.txt" + else + + # call calcdiffs to allow overrides + calcdiffs \ + "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ + checkstyle \ + > "${PATCH_DIR}/diff-checkstyle-${fn}.txt" + fi + + #shellcheck disable=SC2016 + numbranch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + #shellcheck disable=SC2016 + numpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + #shellcheck disable=SC2016 + addpatch=$(wc -l "${PATCH_DIR}/diff-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + + ((fixedpatch=numbranch-numpatch+addpatch)) + + statstring=$(generic_calcdiff_status "${numbranch}" "${numpatch}" "${addpatch}" ) + + mod=${module} + if [[ ${mod} = \. ]]; then + mod=root + fi + + if [[ ${addpatch} -gt 0 ]] ; then + ((result = result + 1)) + module_status ${i} -1 "diff-checkstyle-${fn}.txt" "${mod}: ${BUILDMODEMSG} ${statstring}" + elif [[ ${fixedpatch} -gt 0 ]]; then + module_status ${i} +1 "diff-checkstyle-${fn}.txt" "${mod}: ${BUILDMODEMSG} ${statstring}" + summarize=false + fi + ((i=i+1)) + done + + modules_messages patch checkstyle "${summarize}" + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/cmake.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/cmake.sh b/precommit/src/main/shell/test-patch.d/cmake.sh new file mode 100755 index 0000000..38a48c8 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/cmake.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool cmake + +CMAKE=cmake +CMAKE_BUILD_DIR="build" +CMAKE_ROOT_BUILD=true + +## @description cmake usage hook +## @audience private +## @stability evolving +## @replaceable no +function cmake_usage +{ + yetus_add_option "--cmake-build-dir=" "build directory off of each module to use (default: ${CMAKE_BUILD_DIR})" + yetus_add_option "--cmake-cmd=" "The 'cmake' command to use (default 'cmake')" + yetus_add_option "--cmake-root-build=" "Only build off of root, don't use modules (default: ${CMAKE_ROOT_BUILD})" +} + +## @description cmake argument parser +## @audience private +## @stability evolving +## @param args +function cmake_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --cmake-build-dir=*) + CMAKE_BUILD_DIR=${i#*=} + ;; + --cmake-cmd=*) + CMAKE=${i#*=} + ;; + --cmake-root-build=*) + CMAKE_ROOT_BUILD=${i#*=} + ;; + esac + done +} + +## @description initialize cmake +## @audience private +## @stability evolving +## @replaceable no +function cmake_initialize +{ + if ! declare -f make_executor > /dev/null; then + yetus_error "ERROR: cmake requires make to be enabled." + return 1 + fi +} + +## @description precheck cmake +## @audience private +## @stability evolving +## @replaceable no +function cmake_precheck +{ + declare cmake_version + if ! verify_command cmake "${CMAKE}"; then + add_vote_table -1 cmake "cmake was not available." + return 1 + fi + # finally let folks know what version they'll be dealing with. + cmake_version=$(${CMAKE} --version 2>/dev/null | head -n 1 2>/dev/null) + add_footer_table cmake "version: ${cmake_version}" + + make_precheck +} + + +## @description cmake module manipulation +## @audience private +## @stability evolving +## @replaceable no +function cmake_reorder_modules +{ + if [[ "${CMAKE_ROOT_BUILD}" = true ]]; then + #shellcheck disable=SC2034 + BUILDTOOLCWD="@@@BASEDIR@@@/${CMAKE_BUILD_DIR}" + #shellcheck disable=SC2034 + CHANGED_MODULES=(".") + #shellcheck disable=SC2034 + CHANGED_UNION_MODULES="." + else + #shellcheck disable=SC2034 + BUILDTOOLCWD="@@@MODULEDIR@@@/${CMAKE_BUILD_DIR}" + fi +} + +## @description get the name of the cmake build filename +## @audience private +## @stability evolving +## @replaceable no +## @return cmake build file +function cmake_buildfile +{ + echo "CMakeLists.txt" +} + +## @description get the name of the cmake binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param cmake params +function cmake_executor +{ + if [[ $1 = "CMakeLists.txt" ]]; then + echo "${CMAKE}" + else + make_executor "$@" + fi +} + +## @description precompile for cmake +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +function cmake_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != cmake ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + big_console_header "cmake CMakeLists.txt: ${PATCH_BRANCH}" + else + big_console_header "cmake CMakeLists.txt: ${BUILDMODE}" + fi + + personality_modules "${repostatus}" CMakeLists.txt + + modules_workers "${repostatus}" CMakeLists.txt @@@MODULEDIR@@@ + result=$? + modules_messages "${repostatus}" CMakeLists.txt true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description cmake worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function cmake_modules_worker +{ + make_modules_worker "$@" +} + +## @description cmake module queuer +## @audience private +## @stability evolving +## @replaceable no +function cmake_builtin_personality_modules +{ + make_builtin_personality_modules "$@" +} + +## @description cmake test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function cmake_builtin_personality_file_tests +{ + declare filename=$1 + + if [[ ${filename} =~ CMakeLists.txt$ ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + else + make_builtin_personality_file_tests "${filename}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/ctest.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/ctest.sh b/precommit/src/main/shell/test-patch.d/ctest.sh new file mode 100755 index 0000000..70d91dd --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/ctest.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_format ctest + +## @description initialize ctest +## @audience private +## @stability evolving +## @replaceable no +function ctest_initialize +{ + # shellcheck disable=SC2034 + CTEST_OUTPUT_ON_FAILURE=true +} + +## @description ctest log processor +## @audience private +## @stability evolving +## @replaceable no +## @param module +## @param logfile +## @param filefragment +## @return status +function ctest_process_tests +{ + # shellcheck disable=SC2034 + declare module=$1 + # shellcheck disable=SC2034 + declare buildlogfile=$2 + declare filefrag=$3 + declare result=0 + declare module_failed_tests + declare filenames + declare fn + declare reallog + + # this file contains the lists of tests that failed + # cwd should be the module that we were executing in + # so no need to do anything fancy here + filenames=$(find . -type f -name LastTestsFailed.log) + + if [[ -n "${filenames}" ]]; then + for fn in ${filenames}; do + + # let's record what tests failed and be able to report those + module_failed_tests=$(cut -f2 -d: "${fn}") + CTEST_FAILED_TESTS="${CTEST_FAILED_TESTS} ${module_failed_tests}" + ((result=result+1)) + + # next, let's do some extra log processing + # this file contains the log of the tests that were run + # when the failures happened. it will be in the same dir + # as the lasttestsfailed.log file + reallog=$(dirname "${fn}")/LastTest.log + + # this should always be true, but.... + if [[ -f "${reallog}" ]]; then + module_failed_tests=$(cut -f2 -d: "${fn}") + + # let's copy it to the patchdir so that people can find the failures + # long after the build has been done + cp "${reallog}" "${PATCH_DIR}/patch-${filefrag}-ctest.txt" + CTEST_LOGS=("${CTEST_LOGS[@]}" "@@BASE@@/patch-${filefrag}-ctest.txt") + fi + done + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +## @description cmake test table results +## @audience private +## @stability evolving +## @replaceable no +## @param jdk +function ctest_finalize_results +{ + declare jdk=$1 + declare fn + + if [[ -n "${CTEST_FAILED_TESTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Failed CTEST tests" ${CTEST_FAILED_TESTS} + CTEST_FAILED_TESTS="" + for fn in "${CTEST_LOGS[@]}"; do + add_footer_table "CTEST" "${fn}" + done + CTEST_LOGS=() + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/dupname.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/dupname.sh b/precommit/src/main/shell/test-patch.d/dupname.sh new file mode 100755 index 0000000..6795308 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/dupname.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# dupname check ALWAYS gets activated +add_test_type dupname +add_test dupname + + +## @description Sort an array by its elements, ignoring case +## @audience private +## @stability evolving +## @replaceable yes +## @param arrayvar +function dupname_icase_sort_array +{ + declare arrname=$1 + declare arrref="${arrname}[@]" + declare array=("${!arrref}") + + declare globstatus + declare oifs + declare -a sa + + globstatus=$(set -o | grep noglob | awk '{print $NF}') + + if [[ -n ${IFS} ]]; then + oifs=${IFS} + fi + set -f + # shellcheck disable=SC2034 + IFS=$'\n' sa=($(sort -f <<<"${array[*]}")) + # shellcheck disable=SC1083 + eval "${arrname}"=\(\"\${sa[@]}\"\) + + if [[ -n "${oifs}" ]]; then + IFS=${oifs} + else + unset IFS + fi + + if [[ "${globstatus}" = off ]]; then + set +f + fi +} + +## @description Check the current patchfile for case issues +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function dupname_precheck +{ + declare -i count=0 + declare cur + declare fn + declare prev + declare -a tmpfiles + + tmpfiles=("${CHANGED_FILES[@]}") + + big_console_header "Checking for duplicated filenames that differ only in case" + start_clock + + pushd "${BASEDIR}" >/dev/null || return 1 + + # check the existing tree + for fn in "${CHANGED_FILES[@]}"; do + existing=$(${GIT} ls-files ":(icase)${fn}") + if [[ -n "${existing}" ]]; then + if [[ "${existing}" != "${fn}" ]]; then + echo "patch:${fn} tree:${existing}" >> "${PATCH_DIR}/dupnames.txt" + ((count=count + 1)) + fi + fi + done + + popd >/dev/null || return 1 + + dupname_icase_sort_array tmpfiles + + for cur in "${tmpfiles[@]}"; do + if [[ -n ${prev} ]]; then + if [[ "${cur}" != "${prev}" ]]; then + curlc=$(echo "${cur}" | tr '[:upper:]' '[:lower:]') + if [[ "${curlc}" == "${prevlc}" ]]; then + echo "patch:${cur} patch:${prev}" >> "${PATCH_DIR}/dupnames.txt" + ((count=count + 1)) + fi + fi + fi + prev=${cur} + prevlc=${curlc} + done + + if [[ ${count} -gt 0 ]]; then + if [[ "${BUILDMODE}" != full ]]; then + add_vote_table -1 dupname "The patch has ${count}" \ + " duplicated filenames that differ only in case." + add_footer_table dupname "@@BASE@@/dupnames.txt" + yetus_error "ERROR: Won't apply the patch; may break the workspace." + return 1 + else + add_vote_table -1 dupname "Source has ${count}" \ + " duplicated filenames that differ only in case." + add_footer_table dupname "@@BASE@@/dupnames.txt" + fi + fi + + return 0 +} \ No newline at end of file