From commits-return-1349-archive-asf-public=cust-asf.ponee.io@yetus.apache.org Mon May 20 23:08:49 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 24288180627 for ; Tue, 21 May 2019 01:08:49 +0200 (CEST) Received: (qmail 62715 invoked by uid 500); 20 May 2019 23:08:48 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 62706 invoked by uid 99); 20 May 2019 23:08:48 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 May 2019 23:08:48 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 648B885D03; Mon, 20 May 2019 23:08:48 +0000 (UTC) Date: Mon, 20 May 2019 23:08:48 +0000 To: "commits@yetus.apache.org" Subject: [yetus] branch master updated: YETUS-858. Add support for Golang (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155839372834.31360.7908036620827944700@gitbox.apache.org> From: aw@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: yetus X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: a3800fbeb49a6e17760a78417ede5dcff39cc361 X-Git-Newrev: 3bc18795b28a72c41bb2c7d594f423437ce8d93b X-Git-Rev: 3bc18795b28a72c41bb2c7d594f423437ce8d93b X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/yetus.git The following commit(s) were added to refs/heads/master by this push: new 3bc1879 YETUS-858. Add support for Golang (#57) 3bc1879 is described below commit 3bc18795b28a72c41bb2c7d594f423437ce8d93b Author: Allen Wittenauer AuthorDate: Mon May 20 16:08:43 2019 -0700 YETUS-858. Add support for Golang (#57) --- .../documentation/in-progress/precommit-basic.md | 15 +- precommit/src/main/shell/core.d/00-yetuslib.sh | 35 +++- .../src/main/shell/test-patch-docker/Dockerfile | 15 ++ precommit/src/main/shell/test-patch.d/checkmake.sh | 186 +++++++++++++++++++ precommit/src/main/shell/test-patch.d/golang.sh | 178 ++++++++++++++++++ precommit/src/main/shell/test-patch.d/golangci.sh | 201 +++++++++++++++++++++ precommit/src/main/shell/test-patch.d/make.sh | 2 +- precommit/src/main/shell/test-patch.d/revive.sh | 181 +++++++++++++++++++ .../src/main/shell/test-patch.d/whitespace.sh | 2 +- precommit/src/main/shell/test-patch.sh | 66 +++++-- 10 files changed, 858 insertions(+), 23 deletions(-) diff --git a/asf-site-src/source/documentation/in-progress/precommit-basic.md b/asf-site-src/source/documentation/in-progress/precommit-basic.md index 67a6b3a..9b8be39 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-basic.md +++ b/asf-site-src/source/documentation/in-progress/precommit-basic.md @@ -108,17 +108,28 @@ Unit Test Formats: * [JUnit](http://junit.org/) * [TAP](https://testanything.org/) +Compiler Support: + +* C/C++ +* Go +* Java +* Scala + Language Support, Licensing, and more: -* [Apache Creadur Rat](http://creadur.apache.org/rat/) entries in build system +* [Apache Creadur Rat](http://creadur.apache.org/rat/) entries in build system or installed +* [checkmake](https://github.com/mrtazz/checkmake) installed * [checkstyle](http://checkstyle.sourceforge.net/) entries in build system (ant and maven only) * [FindBugs](http://findbugs.sourceforge.net/) entries in build system and 3.x executables * NOTE: only one of FindBugs or SpotBugs may be used at a time. -* [jshint](https://jshint.com) installed +* [golangci-lint](https://github.com/golangci/golangci-lint) + * NOTE: only Go modules are supported * [hadolint](https://github.com/hadolint/hadolint) installed +* [jshint](https://jshint.com) installed * [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli) installed * [Perl::Critic](http://perlcritic.com/) installed * [pylint](http://www.pylint.org/) installed +* [revive](https://github.com/mgechev/revive) installed * [rubocop](http://batsov.com/rubocop/) installed * [shellcheck](https://github.com/koalaman/shellcheck) installed, preferably 0.3.6 or higher *[SpotBugs](https://spotbugs.github.io/)) entries in build system and 3.x executables diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh b/precommit/src/main/shell/core.d/00-yetuslib.sh index c86bc71..780dc2a 100755 --- a/precommit/src/main/shell/core.d/00-yetuslib.sh +++ b/precommit/src/main/shell/core.d/00-yetuslib.sh @@ -464,6 +464,38 @@ function yetus_sort_and_unique_array fi } +## @description find the deepest entry of a directory array +## @description NOTE: array and filename MUST be absolute paths +## @audience public +## @stability stable +## @replaceable no +## @param array +## @param fn +## @return dir if match +function yetus_find_deepest_directory +{ + declare arrname=$1 + declare arrref="${arrname}[@]" + declare array=("${!arrref}") + declare fn=$2 + + declare d + declare tvalsize + declare val + declare valsize + + for d in "${array[@]}"; do + if yetus_relative_dir "${d}" "${fn}" >/dev/null; then + tvalsize=${d//[^/]/} + if [[ ${#tvalsize} -gt ${valsize} ]]; then + valsize=${#tvalsize} + val=${d} + fi + fi + done + echo "${val}" +} + ## @description Get the date in ctime format ## @audience public ## @stability stable @@ -472,8 +504,7 @@ function yetus_get_ctime { if [[ "${BASH_VERSINFO[0]}" -gt 4 ]] \ || [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -gt 1 ]]; then - # shellcheck disable=SC2183 - printf "%(%s)T" + printf "%(%s)T" -1 else date +"%s" fi diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile index 52ba190..09bac9b 100644 --- a/precommit/src/main/shell/test-patch-docker/Dockerfile +++ b/precommit/src/main/shell/test-patch-docker/Dockerfile @@ -247,6 +247,21 @@ RUN curl -sL https://deb.nodesource.com/setup_11.x | bash - \ && rm -rf /var/lib/apt/lists/* && \ npm install -g jshint@2.10.2 markdownlint-cli@0.15.0 +### +# Install golang and supported helpers +### +# hadolint ignore=DL3008 +RUN add-apt-repository -y ppa:longsleep/golang-backports \ + && apt-get -q update \ + && apt-get -q install --no-install-recommends -y golang-go \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN go get -u github.com/mgechev/revive \ + && go get -u github.com/mrtazz/checkmake \ + && (GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.16.0) \ + && mv /root/go/bin/* /usr/local/bin \ + && rm -rf /root/go + #### # YETUS CUT HERE # Anthing after the above line is ignored by Yetus, so could diff --git a/precommit/src/main/shell/test-patch.d/checkmake.sh b/precommit/src/main/shell/test-patch.d/checkmake.sh new file mode 100755 index 0000000..da570d0 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/checkmake.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +add_test_type checkmake + +CHECKMAKE_TIMER=0 + +CHECKMAKE=${CHECKMAKE:-$(command -v checkmake 2>/dev/null)} + +function checkmake_usage +{ + yetus_add_option "--checkmake=" "path to checkmake executable" + yetus_add_option "--checkmake-config=" "relative path to checkmake config in source tree [default: none]" +} + +function checkmake_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --checkmake=*) + CHECKMAKE=${i#*=} + ;; + --checkmake-config=*) + CHECKMAKE_CONFIG=${i#*=} + ;; + esac + done +} + +function checkmake_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ /Makefile$ ]] || [[ ${filename} =~ ^Makefile$ ]]; then + add_test checkmake + fi +} + +function checkmake_precheck +{ + if ! verify_command checkmake "${CHECKMAKE}"; then + add_vote_table 0 checkmake "checkmake was not available." + delete_test checkmake + fi +} + +function checkmake_exec +{ + declare i + declare repostatus=$1 + declare -a args + + echo "Running checkmake against identified Makefiles." + pushd "${BASEDIR}" >/dev/null || return 1 + + + args=('--format={{.LineNumber}}:{{.Rule}}:{{.Violation}}') + if [[ -f "${CHECKMAKE_CONFIG}" ]]; then + args+=("--config=${CHECKMAKE_CONFIG}") + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ /Makefile$ ]] || [[ ${i} =~ ^Makefile$ ]]; then + if [[ -f ${i} ]]; then + while read -r; do + echo "${i}:${REPLY}" >> "${PATCH_DIR}/${repostatus}-checkmake-result.txt" + done < <("${CHECKMAKE}" "${args[@]}" "${i}") + fi + fi + done + + popd >/dev/null || return 1 + return 0 +} + +function checkmake_preapply +{ + declare i + declare -a args + + if ! verify_needed_test checkmake; then + return 0 + fi + + big_console_header "checkmake plugin: ${PATCH_BRANCH}" + + start_clock + + checkmake_exec branch + + CHECKMAKE_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function checkmake_calcdiffs +{ + column_calcdiffs "$@" +} + +function checkmake_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test checkmake; then + return 0 + fi + + big_console_header "checkmake plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKMAKE_TIMER}" + + checkmake_exec patch + + calcdiffs \ + "${PATCH_DIR}/branch-checkmake-result.txt" \ + "${PATCH_DIR}/patch-checkmake-result.txt" \ + checkmake \ + > "${PATCH_DIR}/diff-patch-checkmake.txt" + diffPostpatch=$("${AWK}" -F: 'BEGIN {sum=0} 3/dev/null) + +declare -a GOMOD_DIRS +GOMOD_DIRS_CONTROL=reset + +## @description Usage info for go plugin +## @audience private +## @stability evolving +## @replaceable no +function golang_usage +{ + yetus_add_option "--golang-go=" "Location of the go binary (default: \"${GOEXE:-not found}\")" +} + +## @description Option parsing for go plugin +## @audience private +## @stability evolving +## @replaceable no +function golang_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --golang-go=*) + GOEXE=${i#*=} + ;; + esac + done +} + +## @description find all non-vendor directories that have a go.mod file +## @audience public +## @stability evolving +## @replaceable yes +function golang_gomod_find +{ + declare input + + if [[ "${GOMOD_DIRS_CONTROL}" == reset ]]; then + GOMOD_DIRS=() + while read -r; do + if [[ ! "${REPLY}" =~ /vendor/ ]] && + [[ ! "${REPLY}" =~ ^vendor ]]; then + input=${REPLY%%/go.mod} + GOMOD_DIRS+=("${input}") + fi + done < <(find "${BASEDIR}" -name go.mod) + GOMOD_DIRS_CONTROL=filled + fi +} + + +## @description Determine if a file is in GOMOD_DIRS[@] +## @audience public +## @stability evolving +## @replaceable yes +## @return all matching dirs +function golang_gomod_file +{ + declare fn=${1} + + yetus_find_deepest_directory GOMOD_DIRS "${BASEDIR}/${fn}" +} + + +## @description discover files to check +## @audience private +## @stability stable +## @replaceable yes +function golang_filefilter +{ + declare filename=$1 + + golang_gomod_find + + if [[ "${filename}" =~ \.(c|h|go|s|cc)$ ]] || + [[ "${filename}" =~ go.mod$ ]]; then + if golang_gomod_file "${filename}" >/dev/null; then + yetus_debug "tests/golang: ${filename}" + add_test golang + add_test compile + fi + fi +} + +## @description check for golang compiler errors +## @audience private +## @stability stable +## @replaceable no +function golang_precompile +{ + GOMOD_DIRS_CONTROL=reset +} + +## @description check for golang compiler errors +## @audience private +## @stability stable +## @replaceable no +function golang_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + if ! verify_needed_test golang; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + generic_postlog_compare compile golang "${multijdkmode}" + fi +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable yes +function golang_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC1117 + "${GREP}" -i -E "^.*\.go\:[[:digit:]]*\:" "${input}" > "${output}" +} + +## @description go post +## @audience private +## @stability evolving +## @replaceable yes +function golang_postapply +{ + if [[ -z "${GOEXE}" ]]; then + # shellcheck disable=SC2016 + version=$("${GOEXE}" version 2>&1 | "${AWK}" '{print $3}' 2>&1) + add_version_data golang "${version#* }" + fi +} + +## @description set volumes and options as appropriate for maven +## @audience private +## @stability evolving +## @replaceable yes +function golang_docker_support +{ + add_docker_env CGO_LDFLAGS + add_docker_env CGO_ENABLED + add_docker_env GO111MODULE + add_docker_env GOPATH +} + + +## @description set volumes and options as appropriate for maven +## @audience private +## @stability evolving +## @replaceable yes +function golang_clean +{ + git_checkout_force +} \ No newline at end of file diff --git a/precommit/src/main/shell/test-patch.d/golangci.sh b/precommit/src/main/shell/test-patch.d/golangci.sh new file mode 100644 index 0000000..462dbb0 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/golangci.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +add_test_type golangcilint + +GOLANGCI_TIMER=0 +GOLANGCI_LINT=$(command -v golangci-lint 2>/dev/null) + +## @description Usage info for slack plugin +## @audience private +## @stability evolving +## @replaceable no +function golangcilint_usage +{ + yetus_add_option "--golangcilint=" "Location of the go binary (default: \"${GOLANGCI_LINT:-not found}\")" + yetus_add_option "--golangcilint-config=" "Location of the config file" +} + +## @description Option parsing for slack plugin +## @audience private +## @stability evolving +## @replaceable no +function golangcilint_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --golangcilint=*) + GOLANGCI_LINT=${i#*=} + ;; + --golangcilint-config=*) + GOLANGCI_CONFIG=${i#*=} + ;; + esac + done +} + +function golangcilint_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.go$ ]]; then + add_test golangcilint + fi +} + +function golangcilint_precheck +{ + if [[ -z "${GOLANGCI_LINT}" ]]; then + add_vote_table 0 golangcilint "golangci-lint was not found." + delete_test golangcilint + fi +} + +function golangcilint_exec +{ + declare i + declare repostatus=$1 + declare -a args + declare -a gargs + + if [[ -f "${EXCLUDE_PATHS_FILE}" ]]; then + gargs=("${GREP}" "-v" "-E" "-f" "${EXCLUDE_PATHS_FILE}") + else + gargs=("cat") + fi + + args=("--color=never") + args+=("--out-format=line-number") + args+=("--print-issued-lines=false") + + if [[ -f "${GOLANGCI_CONFIG}" ]]; then + args+=("--config" "${GOLANGCI_CONFIG}") + fi + + golang_gomod_find + + for d in "${GOMOD_DIRS[@]}"; do + pushd "${d}" >/dev/null || return 1 + while read -r; do + p=$(yetus_relative_dir "${BASEDIR}" "${d}") + if [[ -n "${p}" ]]; then + p="${p}/" + fi + echo "${p}${REPLY}" >> "${PATCH_DIR}/${repostatus}-golangcilint-result.txt" + done < <("${GOLANGCI_LINT}" run "${args[@]}" ./... 2>&1 \ + | "${gargs[@]}" \ + | sort -t : -k 1,1 -k 2,2n -k 3,3n) + popd >/dev/null || return 1 + done + return 0 +} + +function golangcilint_preapply +{ + declare i + + if ! verify_needed_test golangcilint; then + return 0 + fi + + big_console_header "golangcilint plugin: ${PATCH_BRANCH}" + + start_clock + + golangcilint_exec branch + GOLANGCI_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function golangcilint_calcdiffs +{ + column_calcdiffs "$@" +} + +function golangcilint_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test golangcilint; then + return 0 + fi + + big_console_header "golangcilint plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${GOLANGCI_TIMER}" + + golangcilint_exec patch + + calcdiffs \ + "${PATCH_DIR}/branch-golangcilint-result.txt" \ + "${PATCH_DIR}/patch-golangcilint-result.txt" \ + golangcilint \ + > "${PATCH_DIR}/diff-patch-golangcilint.txt" + diffPostpatch=$("${AWK}" -F: 'BEGIN {sum=0} 3/dev/null)} + +function revive_usage +{ + yetus_add_option "--revive=" "path to revive executable" + yetus_add_option "--revive-config=" "relative path to revive config in source tree [default: none]" +} + +function revive_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --revive=*) + REVIVE=${i#*=} + ;; + --revive-config=*) + REVIVE_CONFIG=${i#*=} + ;; + esac + done +} + +function revive_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.go$ ]]; then + add_test revive + fi +} + +function revive_precheck +{ + if ! verify_command revive "${REVIVE}"; then + add_vote_table 0 revive "revive was not available." + delete_test revive + fi +} + +function revive_exec +{ + declare i + declare repostatus=$1 + declare -a args + + echo "Running revive against identified go files." + pushd "${BASEDIR}" >/dev/null || return 1 + + args=('-formatter' 'default') + if [[ -f "${REVIVE_CONFIG}" ]]; then + args+=('-config' "${REVIVE_CONFIG}") + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.go$ && -f ${i} ]]; then + "${REVIVE}" "${args[@]}" "${i}" | sort -t : -k1,1 -k2,2n -k3,3n >> "${PATCH_DIR}/${repostatus}-revive-result.txt" + fi + done + + popd >/dev/null || return 1 + return 0 +} + +function revive_preapply +{ + declare i + declare -a args + + if ! verify_needed_test revive; then + return 0 + fi + + big_console_header "revive plugin: ${PATCH_BRANCH}" + + start_clock + + revive_exec branch + + REVIVE_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function revive_calcdiffs +{ + column_calcdiffs "$@" +} + +function revive_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test revive; then + return 0 + fi + + big_console_header "revive plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${REVIVE_TIMER}" + + revive_exec patch + + calcdiffs \ + "${PATCH_DIR}/branch-revive-result.txt" \ + "${PATCH_DIR}/patch-revive-result.txt" \ + revive \ + > "${PATCH_DIR}/diff-patch-revive.txt" + diffPostpatch=$("${AWK}" -F: 'BEGIN {sum=0} 3/dev/null; then + # we need to empty out PATCH_DIR, but + # leave _directories_ in case we are in # re-exec mode (which places a directory full of stuff in it) yetus_debug "Exempting ${exemptdir} from clean" rm "${PATCH_DIR}/*" 2>/dev/null - "${GIT}" clean -xdf -e "${exemptdir}" - status=$? - fi - - if [[ ${status} != 0 ]]; then - yetus_error "ERROR: git clean is failing" - cleanup_and_exit 1 fi if [[ "${GIT_SHALLOW}" == false ]]; then @@ -1244,7 +1276,7 @@ function git_checkout fi fi - if ! "${GIT}" clean -df; then + if ! git_clean; then yetus_error "ERROR: git clean is failing" cleanup_and_exit 1 fi @@ -2971,7 +3003,7 @@ function distclean for plugin in "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do if declare -f "${plugin}_clean" >/dev/null 2>&1; then - yetus_debug "Running ${plugin}_distclean" + yetus_debug "Running ${plugin}_clean" if ! "${plugin}_clean"; then ((result = result+1)) fi