From commits-return-1188-archive-asf-public=cust-asf.ponee.io@yetus.apache.org Mon Dec 10 07:59:55 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 137F618067C for ; Mon, 10 Dec 2018 07:59:54 +0100 (CET) Received: (qmail 5737 invoked by uid 500); 10 Dec 2018 06:59:54 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 5667 invoked by uid 99); 10 Dec 2018 06:59:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Dec 2018 06:59:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7EAD4E131B; Mon, 10 Dec 2018 06:59:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: commits@yetus.apache.org Date: Mon, 10 Dec 2018 06:59:54 -0000 Message-Id: <91b5b8e43a2446eeaa771c7d602f9e74@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/5] yetus git commit: YETUS-713. pylint: rewrite to make it easier to maintain and fix stderr output in brief report YETUS-713. pylint: rewrite to make it easier to maintain and fix stderr output in brief report Signed-off-by: Allen Wittenauer Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/8a502dc0 Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/8a502dc0 Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/8a502dc0 Branch: refs/heads/master Commit: 8a502dc055b7a133a24bca45eb429f43f29834e8 Parents: 671b88f Author: Allen Wittenauer Authored: Mon Nov 12 15:24:00 2018 -0800 Committer: Allen Wittenauer Committed: Sun Dec 9 22:25:43 2018 -0800 ---------------------------------------------------------------------- .../src/main/shell/test-patch.d/briefreport.sh | 34 +++--- precommit/src/main/shell/test-patch.d/pylint.sh | 110 ++++++++++--------- 2 files changed, 79 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/briefreport.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/briefreport.sh b/precommit/src/main/shell/test-patch.d/briefreport.sh index e536b97..caeaaf0 100755 --- a/precommit/src/main/shell/test-patch.d/briefreport.sh +++ b/precommit/src/main/shell/test-patch.d/briefreport.sh @@ -96,6 +96,7 @@ function briefreport_finalreport declare hours declare newtime declare havelogs=false + declare fn if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then return @@ -103,11 +104,10 @@ function briefreport_finalreport big_console_header "Writing Brief Text Report to ${BRIEFOUT_REPORTFILE}" - if [[ ${result} == 0 ]]; then - printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + printf '\n\n+1 overall\n\n' > "${BRIEFOUT_REPORTFILE}" else - printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + printf '\n\n-1 overall\n\n' > "${BRIEFOUT_REPORTFILE}" fi i=0 @@ -137,11 +137,14 @@ function briefreport_finalreport ((i=i+1)) done - tmparray=($(printf "%s\n" "${failed[@]}" | sort -u)) + #shellcheck disable=SC2207 + tmparray=($(printf '%s\n' "${failed[@]}" | sort -u)) failed=("${tmparray[@]}") - tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u)) + #shellcheck disable=SC2207 + tmparray=($(printf '%s\n' "${filtered[@]}" | sort -u)) filtered=("${tmparray[@]}") - tmparray=($(printf "%s\n" "${long[@]}" | sort -u)) + #shellcheck disable=SC2207 + tmparray=($(printf '%s\n' "${long[@]}" | sort -u)) long=("${tmparray[@]}") if [[ ${#failed[@]} -gt 0 ]]; then @@ -198,11 +201,11 @@ function briefreport_finalreport { if [[ -n "${vote// }" ]]; then echo "" - printf " %s:\n" "${vote}" + printf ' %s:\n' "${vote}" echo "" vote="" fi - printf " %s\n" "${subs}" + printf ' %s\n' "${subs}" } >> "${BRIEFOUT_REPORTFILE}" ((i=i+1)) done @@ -231,22 +234,25 @@ function briefreport_finalreport subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|) comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f3 -d\| | - ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") - # shellcheck disable=SC2016 - size=$(du -sh "${comment// }" | ${AWK} '{print $1}') + "${SED}" -e "s,@@BASE@@,${PATCH_DIR},g") + fn="${comment// }" + if [[ -f ${fn} ]]; then + # shellcheck disable=SC2016 + size=$(du -sh "${fn}" | "${AWK}" '{print $1}') + fi if [[ -n "${BUILD_URL}" ]]; then comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f3 -d\| | - ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") fi { if [[ "${subs}" != "${vote}" ]]; then echo "" - printf " %s:\n" "${subs// }" + printf ' %s:\n' "${subs// }" echo "" vote=${subs} fi - printf " %s [%s]\n" "${comment}" "${size}" + printf ' %s [%s]\n' "${comment}" "${size}" } >> "${BRIEFOUT_REPORTFILE}" fi ((i=i+1)) http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/pylint.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/pylint.sh b/precommit/src/main/shell/test-patch.d/pylint.sh index facba6c..d5a0848 100755 --- a/precommit/src/main/shell/test-patch.d/pylint.sh +++ b/precommit/src/main/shell/test-patch.d/pylint.sh @@ -14,17 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +# SHELLDOC-IGNORE + add_test_type pylint PYLINT_TIMER=0 -PYLINT=${PYLINT:-$(which pylint 2>/dev/null)} +PYLINT=${PYLINT:-$(command -v pylint 2>/dev/null)} +# backward compatibility, do not use PYLINT_OPTIONS=${PYLINT_OPTIONS:-} function pylint_usage { yetus_add_option "--pylint=" "path to pylint executable" - yetus_add_option "--pylint-options=" "pylint options other than output-format and reports" + yetus_add_option "--pylint-rcfile=" "pylint configuration file" } function pylint_parse_args @@ -36,7 +39,11 @@ function pylint_parse_args --pylint=*) PYLINT=${i#*=} ;; + --pylint-rcfile=*) + PYLINT_RCFILE=${i#*=} + ;; --pylint-options=*) + # backward compatibility PYLINT_OPTIONS=${i#*=} ;; esac @@ -60,52 +67,80 @@ function pylint_precheck fi } - -function pylint_preapply +function pylint_executor { - local i - local count - local pylintStderr=branch-pylint-stderr.txt + declare repostatus=$1 + declare i + declare count + declare pylintStderr=${repostatus}-pylint-stderr.txt if ! verify_needed_test pylint; then return 0 fi - big_console_header "pylint plugin: ${PATCH_BRANCH}" + big_console_header "pylint plugin: ${BUILDMODE}" start_clock + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${PYLINT_TIMER}" + + # backward compatibility + # shellcheck disable=SC2206 + pylintopts=(${PYLINT_OPTIONS}) + + if [[ -n "${PYLINT_RCFILE}" ]] && [[ -f "${PYLINT_RCFILE}" ]]; then + pylintopts+=('--rcfile='"${PYLINT_RCFILE}") + fi + + pylintops+=('--persistent=n') + pylintops+=('--reports=n') + pylintops+=('--score=n') + echo "Running pylint against identified python scripts." - pushd "${BASEDIR}" >/dev/null + pushd "${BASEDIR}" >/dev/null || return 1 for i in "${CHANGED_FILES[@]}"; do if [[ ${i} =~ \.py$ && -f ${i} ]]; then - # shellcheck disable=SC2086 - eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \ - 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1> "${PATCH_DIR}/branch-pylint-result.txt" + "${PYLINT}" "${pylintopts[@]}" --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' "${i}" \ + 2>>"${PATCH_DIR}/${pylintStderr}" | "${AWK}" '1> "${PATCH_DIR}/${repostatus}-pylint-result.txt" fi done if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then - count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}") + count=$("${GREP}" -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}") if [[ ${count} -gt 0 ]]; then - add_footer_table pylint "${PATCH_BRANCH} stderr: @@BASE@@/${pylintStderr}" + add_vote_table -1 pylint "Error running pylint. Please check pylint stderr files." + add_footer_table pylint "@@BASE@@/${pylintStderr}" return 1 fi fi rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null - popd >/dev/null + popd >/dev/null || return 1 + return 0 +} + + +function pylint_preapply +{ + declare retval + + if ! verify_needed_test pylint; then + return 0 + fi + + pylint_executor "branch" + retval=$? + # keep track of how much as elapsed for us already PYLINT_TIMER=$(stop_clock) - return 0 + return ${retval} } function pylint_postapply { - declare i - declare count declare numPrepatch declare numPostpatch declare diffPostpatch - declare pylintStderr=patch-pylint-stderr.txt declare fixedpatch declare statstring @@ -113,46 +148,19 @@ function pylint_postapply return 0 fi - big_console_header "pylint plugin: ${BUILDMODE}" - - start_clock - - # add our previous elapsed to our new timer - # by setting the clock back - offset_clock "${PYLINT_TIMER}" - - echo "Running pylint against identified python scripts." - # we re-check this in case one has been added - pushd "${BASEDIR}" >/dev/null - for i in "${CHANGED_FILES[@]}"; do - if [[ ${i} =~ \.py$ && -f ${i} ]]; then - # shellcheck disable=SC2086 - eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \ - 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1> "${PATCH_DIR}/patch-pylint-result.txt" - fi - done - if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then - count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}") - if [[ ${count} -gt 0 ]]; then - add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please check pylint stderr files." - add_footer_table pylint "${BUILDMODEMSG} stderr: @@BASE@@/${pylintStderr}" - return 1 - fi - fi - rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null - popd >/dev/null + pylint_executor patch # shellcheck disable=SC2016 - PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}') + PYLINT_VERSION=$("${PYLINT}" --version 2>/dev/null | "${GREP}" pylint | "${AWK}" '{print $NF}') add_footer_table pylint "v${PYLINT_VERSION%,}" calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" \ "${PATCH_DIR}/patch-pylint-result.txt" \ pylint > "${PATCH_DIR}/diff-patch-pylint.txt" - numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt") - numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt") + numPrepatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/branch-pylint-result.txt") + numPostpatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/patch-pylint-result.txt") # Exclude Pylint messages from the information category to avoid false positives (see YETUS-309). - diffPostpatch=$(${GREP} -c "^.*:.*: \[[^I].*\] " "${PATCH_DIR}/diff-patch-pylint.txt") + diffPostpatch=$("${GREP}" -c '^.*:.*: \[[^I].*\] ' "${PATCH_DIR}/diff-patch-pylint.txt") ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))