yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [2/5] yetus git commit: YETUS-713. pylint: rewrite to make it easier to maintain and fix stderr output in brief report
Date Mon, 10 Dec 2018 06:59:54 GMT
YETUS-713. pylint: rewrite to make it easier to maintain and fix stderr output in brief report

Signed-off-by: Allen Wittenauer <aw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/8a502dc0
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/8a502dc0
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/8a502dc0

Branch: refs/heads/master
Commit: 8a502dc055b7a133a24bca45eb429f43f29834e8
Parents: 671b88f
Author: Allen Wittenauer <aw@apache.org>
Authored: Mon Nov 12 15:24:00 2018 -0800
Committer: Allen Wittenauer <aw@apache.org>
Committed: Sun Dec 9 22:25:43 2018 -0800

----------------------------------------------------------------------
 .../src/main/shell/test-patch.d/briefreport.sh  |  34 +++---
 precommit/src/main/shell/test-patch.d/pylint.sh | 110 ++++++++++---------
 2 files changed, 79 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/briefreport.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/briefreport.sh b/precommit/src/main/shell/test-patch.d/briefreport.sh
index e536b97..caeaaf0 100755
--- a/precommit/src/main/shell/test-patch.d/briefreport.sh
+++ b/precommit/src/main/shell/test-patch.d/briefreport.sh
@@ -96,6 +96,7 @@ function briefreport_finalreport
   declare hours
   declare newtime
   declare havelogs=false
+  declare fn
 
   if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then
     return
@@ -103,11 +104,10 @@ function briefreport_finalreport
 
   big_console_header "Writing Brief Text Report to ${BRIEFOUT_REPORTFILE}"
 
-
   if [[ ${result} == 0 ]]; then
-    printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+    printf '\n\n+1 overall\n\n' > "${BRIEFOUT_REPORTFILE}"
   else
-    printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+    printf '\n\n-1 overall\n\n' > "${BRIEFOUT_REPORTFILE}"
   fi
 
   i=0
@@ -137,11 +137,14 @@ function briefreport_finalreport
     ((i=i+1))
   done
 
-  tmparray=($(printf "%s\n" "${failed[@]}" | sort -u))
+  #shellcheck disable=SC2207
+  tmparray=($(printf '%s\n' "${failed[@]}" | sort -u))
   failed=("${tmparray[@]}")
-  tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u))
+  #shellcheck disable=SC2207
+  tmparray=($(printf '%s\n' "${filtered[@]}" | sort -u))
   filtered=("${tmparray[@]}")
-  tmparray=($(printf "%s\n" "${long[@]}" | sort -u))
+  #shellcheck disable=SC2207
+  tmparray=($(printf '%s\n' "${long[@]}" | sort -u))
   long=("${tmparray[@]}")
 
   if [[ ${#failed[@]} -gt 0 ]]; then
@@ -198,11 +201,11 @@ function briefreport_finalreport
       {
         if [[ -n "${vote// }" ]]; then
           echo ""
-          printf "   %s:\n" "${vote}"
+          printf '   %s:\n' "${vote}"
           echo ""
           vote=""
         fi
-        printf "      %s\n" "${subs}"
+        printf '      %s\n' "${subs}"
       } >> "${BRIEFOUT_REPORTFILE}"
       ((i=i+1))
     done
@@ -231,22 +234,25 @@ function briefreport_finalreport
         subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|)
         comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
                     cut -f3 -d\| |
-                    ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
-        # shellcheck disable=SC2016
-        size=$(du -sh "${comment// }" | ${AWK} '{print $1}')
+                    "${SED}" -e "s,@@BASE@@,${PATCH_DIR},g")
+        fn="${comment// }"
+        if [[ -f ${fn} ]]; then
+          # shellcheck disable=SC2016
+          size=$(du -sh "${fn}" | "${AWK}" '{print $1}')
+        fi
         if [[ -n "${BUILD_URL}" ]]; then
           comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
                     cut -f3 -d\| |
-                    ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+                    "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
         fi
         {
           if [[ "${subs}" != "${vote}" ]]; then
             echo ""
-            printf "   %s:\n" "${subs// }"
+            printf '   %s:\n' "${subs// }"
             echo ""
             vote=${subs}
           fi
-          printf "      %s [%s]\n" "${comment}" "${size}"
+          printf '      %s [%s]\n' "${comment}" "${size}"
         } >> "${BRIEFOUT_REPORTFILE}"
       fi
       ((i=i+1))

http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/pylint.sh b/precommit/src/main/shell/test-patch.d/pylint.sh
index facba6c..d5a0848 100755
--- a/precommit/src/main/shell/test-patch.d/pylint.sh
+++ b/precommit/src/main/shell/test-patch.d/pylint.sh
@@ -14,17 +14,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# SHELLDOC-IGNORE
+
 add_test_type pylint
 
 PYLINT_TIMER=0
 
-PYLINT=${PYLINT:-$(which pylint 2>/dev/null)}
+PYLINT=${PYLINT:-$(command -v pylint 2>/dev/null)}
+# backward compatibility, do not use
 PYLINT_OPTIONS=${PYLINT_OPTIONS:-}
 
 function pylint_usage
 {
   yetus_add_option "--pylint=<path>" "path to pylint executable"
-  yetus_add_option "--pylint-options=<path>" "pylint options other than output-format
and reports"
+  yetus_add_option "--pylint-rcfile=<path>" "pylint configuration file"
 }
 
 function pylint_parse_args
@@ -36,7 +39,11 @@ function pylint_parse_args
     --pylint=*)
       PYLINT=${i#*=}
     ;;
+    --pylint-rcfile=*)
+      PYLINT_RCFILE=${i#*=}
+    ;;
     --pylint-options=*)
+      # backward compatibility
       PYLINT_OPTIONS=${i#*=}
     ;;
     esac
@@ -60,52 +67,80 @@ function pylint_precheck
   fi
 }
 
-
-function pylint_preapply
+function pylint_executor
 {
-  local i
-  local count
-  local pylintStderr=branch-pylint-stderr.txt
+  declare repostatus=$1
+  declare i
+  declare count
+  declare pylintStderr=${repostatus}-pylint-stderr.txt
 
   if ! verify_needed_test pylint; then
     return 0
   fi
 
-  big_console_header "pylint plugin: ${PATCH_BRANCH}"
+  big_console_header "pylint plugin: ${BUILDMODE}"
 
   start_clock
 
+  # add our previous elapsed to our new timer
+  # by setting the clock back
+  offset_clock "${PYLINT_TIMER}"
+
+  # backward compatibility
+  # shellcheck disable=SC2206
+  pylintopts=(${PYLINT_OPTIONS})
+
+  if [[ -n "${PYLINT_RCFILE}" ]] && [[ -f "${PYLINT_RCFILE}" ]]; then
+    pylintopts+=('--rcfile='"${PYLINT_RCFILE}")
+  fi
+
+  pylintops+=('--persistent=n')
+  pylintops+=('--reports=n')
+  pylintops+=('--score=n')
+
   echo "Running pylint against identified python scripts."
-  pushd "${BASEDIR}" >/dev/null
+  pushd "${BASEDIR}" >/dev/null || return 1
   for i in "${CHANGED_FILES[@]}"; do
     if [[ ${i} =~ \.py$ && -f ${i} ]]; then
-      # shellcheck disable=SC2086
-      eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}),
{obj}] {msg}' --reports=n ${i}" \
-        2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt"
+      "${PYLINT}" "${pylintopts[@]}" --msg-template='{path}:{line}: [{msg_id}({symbol}),
{obj}] {msg}' "${i}" \
+        2>>"${PATCH_DIR}/${pylintStderr}" | "${AWK}" '1<NR' >> "${PATCH_DIR}/${repostatus}-pylint-result.txt"
     fi
   done
   if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then
-    count=$(${GREP}  -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
+    count=$("${GREP}"  -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
     if [[ ${count} -gt 0 ]]; then
-      add_footer_table pylint "${PATCH_BRANCH} stderr: @@BASE@@/${pylintStderr}"
+      add_vote_table -1 pylint "Error running pylint. Please check pylint stderr files."
+      add_footer_table pylint "@@BASE@@/${pylintStderr}"
       return 1
     fi
   fi
   rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null
-  popd >/dev/null
+  popd >/dev/null || return 1
+  return 0
+}
+
+
+function pylint_preapply
+{
+  declare retval
+
+  if ! verify_needed_test pylint; then
+    return 0
+  fi
+
+  pylint_executor "branch"
+  retval=$?
+
   # keep track of how much as elapsed for us already
   PYLINT_TIMER=$(stop_clock)
-  return 0
+  return ${retval}
 }
 
 function pylint_postapply
 {
-  declare i
-  declare count
   declare numPrepatch
   declare numPostpatch
   declare diffPostpatch
-  declare pylintStderr=patch-pylint-stderr.txt
   declare fixedpatch
   declare statstring
 
@@ -113,46 +148,19 @@ function pylint_postapply
     return 0
   fi
 
-  big_console_header "pylint plugin: ${BUILDMODE}"
-
-  start_clock
-
-  # add our previous elapsed to our new timer
-  # by setting the clock back
-  offset_clock "${PYLINT_TIMER}"
-
-  echo "Running pylint against identified python scripts."
-  # we re-check this in case one has been added
-  pushd "${BASEDIR}" >/dev/null
-  for i in "${CHANGED_FILES[@]}"; do
-    if [[ ${i} =~ \.py$ && -f ${i} ]]; then
-      # shellcheck disable=SC2086
-      eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}),
{obj}] {msg}' --reports=n ${i}" \
-        2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt"
-    fi
-  done
-  if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then
-    count=$(${GREP}  -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
-    if [[ ${count} -gt 0 ]]; then
-      add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please
check pylint stderr files."
-      add_footer_table pylint "${BUILDMODEMSG} stderr: @@BASE@@/${pylintStderr}"
-      return 1
-    fi
-  fi
-  rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null
-  popd >/dev/null
+  pylint_executor patch
 
   # shellcheck disable=SC2016
-  PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}')
+  PYLINT_VERSION=$("${PYLINT}" --version 2>/dev/null | "${GREP}" pylint | "${AWK}" '{print
$NF}')
   add_footer_table pylint "v${PYLINT_VERSION%,}"
 
   calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" \
             "${PATCH_DIR}/patch-pylint-result.txt" \
             pylint > "${PATCH_DIR}/diff-patch-pylint.txt"
-  numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt")
-  numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt")
+  numPrepatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/branch-pylint-result.txt")
+  numPostpatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/patch-pylint-result.txt")
   # Exclude Pylint messages from the information category to avoid false positives (see YETUS-309).
-  diffPostpatch=$(${GREP} -c "^.*:.*: \[[^I].*\] " "${PATCH_DIR}/diff-patch-pylint.txt")
+  diffPostpatch=$("${GREP}" -c '^.*:.*: \[[^I].*\] ' "${PATCH_DIR}/diff-patch-pylint.txt")
 
   ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))
 


Mime
View raw message