yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject yetus git commit: YETUS-640. add hadolint support
Date Fri, 24 Aug 2018 00:06:21 GMT
Repository: yetus
Updated Branches:
  refs/heads/master 5a2c02552 -> 7615f35f9


YETUS-640. add hadolint support

Signed-off-by: Jack Bearden <jack@jackbearden.com>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/7615f35f
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/7615f35f
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/7615f35f

Branch: refs/heads/master
Commit: 7615f35f9835fbe144697b6c89e313d70316e99e
Parents: 5a2c025
Author: Allen Wittenauer <aw@apache.org>
Authored: Tue Jul 10 16:06:02 2018 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Thu Aug 23 17:05:44 2018 -0700

----------------------------------------------------------------------
 .hadolint.yaml                         |  17 +++
 precommit/test-patch-docker/Dockerfile |  87 ++++++++----
 precommit/test-patch.d/hadolint.sh     | 204 ++++++++++++++++++++++++++++
 3 files changed, 284 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/.hadolint.yaml
----------------------------------------------------------------------
diff --git a/.hadolint.yaml b/.hadolint.yaml
new file mode 100644
index 0000000..6645586
--- /dev/null
+++ b/.hadolint.yaml
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ignored:
+  - DL3008

http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile
index 2f54bda..8494d5c 100644
--- a/precommit/test-patch-docker/Dockerfile
+++ b/precommit/test-patch-docker/Dockerfile
@@ -26,6 +26,7 @@
 FROM ubuntu:xenial
 
 WORKDIR /root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
@@ -41,7 +42,9 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends
-y \
     pinentry-curses \
     pkg-config \
     rsync \
-    software-properties-common
+    software-properties-common \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ###
 # Set the locale
@@ -58,16 +61,20 @@ ENV LC_ALL en_US.UTF-8
 ####
 # OpenJDK 8
 ####
-RUN apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless
\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 #######
 # OpenJDK 9
 # w/workaround for
 # https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
 #######
-RUN apt-get -o Dpkg::Options::="--force-overwrite" \
+RUN apt-get -q update && apt-get -o Dpkg::Options::="--force-overwrite" \
     -q install --no-install-recommends -y \
-    openjdk-9-jdk-headless
+    openjdk-9-jdk-headless \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 #######
 # Set default Java
@@ -78,72 +85,91 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" \
 #
 RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
 RUN update-alternatives --get-selections | grep -i jdk | \
-    while read line; do \
-      alternative=$(echo $line | awk '{print $1}'); \
-      path=$(echo $line | awk '{print $3}'); \
-      newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
-      update-alternatives --set $alternative $newpath; \
+    while read -r line; do \
+      alternative=$(echo "$line" | awk '{print $1}'); \
+      path=$(echo "$line" | awk '{print $3}'); \
+      newpath=${path//java-9/java-8}; \
+      update-alternatives --set "$alternative" "$newpath"; \
     done
 
 
 ####
 # Install ant
 ####
-RUN apt-get -q install --no-install-recommends -y ant
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ####
 # Install GNU automake, GNU make, and related
 ####
-RUN apt-get -q install --no-install-recommends -y autoconf automake libtool make
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y autoconf automake
libtool make \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ####
 # Install bats (TAP-capable unit testing for shell scripts)
 ####
-RUN apt-get -q install --no-install-recommends -y bats
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y bats \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ####
 # Install cmake
 ####
-RUN apt-get -q install --no-install-recommends -y cmake
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ######
 # Install findbugs
 ######
-RUN apt-get -q install --no-install-recommends -y findbugs
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 ENV FINDBUGS_HOME /usr
 
 ####
 # Install GNU C/C++
 ####
-RUN apt-get -q install --no-install-recommends -y g++ gcc libc-dev
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y g++ gcc libc-dev
\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ######
 # Install maven
 ######
-RUN apt-get -q install --no-install-recommends -y maven
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y maven \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ######
 # Install perl
 ######
-RUN apt-get -q install --no-install-recommends -y perl libperl-critic-perl
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y perl libperl-critic-perl
\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ######
 # Install python and pylint
 ######
-RUN apt-get -q install --no-install-recommends -y python \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \
     python2.7 \
     python-pip \
     python-pkg-resources \
     python-setuptools \
-    python-wheel
-RUN pip2 install --upgrade pip==9.0.3
-RUN pip install -v pylint==1.9.2
+    python-wheel \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+RUN pip2 install -v pylint==1.9.2
 
 ####
 # Install ruby, rubocop, and ruby-lint
 ###
 RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc
-RUN apt-get -q install --no-install-recommends -y ruby ruby-dev
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y ruby ruby-dev
\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 RUN gem install rake
 RUN gem install rubocop
 RUN gem install ruby-lint
@@ -152,8 +178,19 @@ RUN gem install ruby-lint
 # Install shellcheck (shell script lint)
 ####
 RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck
\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
+###
+# Install hadolint
+####
+RUN curl -L -s -S \
+   https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
+   -o /bin/hadolint && \
+   chmod a+rx /bin/hadolint && \
+   shasum -a 512 /bin/hadolint | \
+   awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca"
{exit(1)}'
 
 ####
 # YETUS CUT HERE
@@ -161,4 +198,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends
-y shellchec
 # include other requirements not needed by your development
 # (but not build) environment
 ###
-RUN apt-get -q install --no-install-recommends -y sudo vim
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y sudo vim \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch.d/hadolint.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/hadolint.sh b/precommit/test-patch.d/hadolint.sh
new file mode 100755
index 0000000..2392e29
--- /dev/null
+++ b/precommit/test-patch.d/hadolint.sh
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+add_test_type hadolint
+
+HADOLINT_TIMER=0
+HADOLINT=${HADOLINT:-$(command -v hadolint 2>/dev/null)}
+
+# files that are going to get hadolint'd
+HADOLINT_CHECKFILES=()
+
+function hadolint_filefilter
+{
+  declare filename=$1
+
+  if [[ ${filename} =~ Dockerfile$ ]]; then
+    add_test hadolint
+    yetus_add_array_element HADOLINT_CHECKFILES "${filename}"
+  fi
+}
+
+function hadolint_precheck
+{
+  declare langs
+
+  if ! verify_command "hadolint" "${HADOLINT}"; then
+    add_vote_table 0 hadolint "hadolint was not available."
+    delete_test hadolint
+  fi
+
+  if [[ -z "${LANG}" ]]; then
+    langs=$(locale -a)
+    if [[ ${langs}  =~ C.UTF-8 ]]; then
+      yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing C.UTF-8."
+      export LANG=C.UTF-8
+      export LC_ALL=C.UTF-8
+    elif [[ ${langs}  =~ en_US.UTF-8 ]]; then
+      yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing en_US.UTF-8."
+      export LANG=en_US.UTF-8
+      export LC_ALL=en_US.UTF-8
+    else
+      for i in ${langs}; do
+        if [[ "${i}" =~ UTF-8 ]]; then
+          yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing ${i}."
+          export LANG="${i}"
+          export LC_ALL="${i}"
+          break
+        fi
+      done
+    fi
+  fi
+
+  if [[ ! "${LANG}" =~ UTF-8 ]]; then
+    yetus_error "WARNING: hadolint may fail without UTF-8 locale setting."
+  fi
+}
+
+function hadolint_logic
+{
+  declare repostatus=$1
+  declare i
+
+  pushd "${BASEDIR}" >/dev/null || return 1
+
+  for i in "${HADOLINT_CHECKFILES[@]}"; do
+    if [[ -f "${i}" ]]; then
+      echo " * ${i}"
+      "${HADOLINT}" "${i}" >> "${PATCH_DIR}/${repostatus}-hadolint-result.txt"
+    fi
+  done
+  popd > /dev/null || return 1
+}
+
+function hadolint_preapply
+{
+  if ! verify_needed_test hadolint; then
+    return 0
+  fi
+
+  big_console_header "hadolint plugin: ${PATCH_BRANCH}"
+
+  start_clock
+
+  hadolint_logic branch
+
+  # keep track of how much as elapsed for us already
+  HADOLINT_TIMER=$(stop_clock)
+  return 0
+}
+
+## filename:line\sCODE Text
+function hadolint_calcdiffs
+{
+  declare branch=$1
+  declare patch=$2
+  declare tmp=${PATCH_DIR}/pl.$$.${RANDOM}
+  declare j
+
+  # first, pull out just the errors
+  # shellcheck disable=SC2016
+  ${AWK} -F: '{print $NF}' "${branch}" | cut -d' ' -f2- > "${tmp}.branch"
+
+  # shellcheck disable=SC2016
+  ${AWK} -F: '{print $NF}' "${patch}" | cut -d' ' -f2- > "${tmp}.patch"
+
+  ${DIFF} --unchanged-line-format="" \
+     --old-line-format="" \
+     --new-line-format="%dn " \
+     "${tmp}.branch" \
+     "${tmp}.patch" > "${tmp}.lined"
+
+  # now, pull out those lines of the raw output
+  # shellcheck disable=SC2013
+  for j in $(cat "${tmp}.lined"); do
+    # shellcheck disable=SC2086
+    head -${j} "${patch}" | tail -1
+  done
+
+  rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
+}
+
+function hadolint_postapply
+{
+  declare i
+  declare numPrepatch
+  declare numPostpatch
+  declare diffPostpatch
+  declare fixedpatch
+  declare statstring
+
+  if ! verify_needed_test hadolint; then
+    return 0
+  fi
+
+  big_console_header "hadolint plugin: ${BUILDMODE}"
+
+  start_clock
+
+  # add our previous elapsed to our new timer
+  # by setting the clock back
+  offset_clock "${HADOLINT_TIMER}"
+
+  hadolint_logic patch
+
+  calcdiffs \
+    "${PATCH_DIR}/branch-hadolint-result.txt" \
+    "${PATCH_DIR}/patch-hadolint-result.txt" \
+    hadolint \
+      > "${PATCH_DIR}/diff-patch-hadolint.txt"
+
+  # shellcheck disable=SC2016
+  numPrepatch=$(wc -l "${PATCH_DIR}/branch-hadolint-result.txt" | ${AWK} '{print $1}')
+
+  # shellcheck disable=SC2016
+  numPostpatch=$(wc -l "${PATCH_DIR}/patch-hadolint-result.txt" | ${AWK} '{print $1}')
+
+  # shellcheck disable=SC2016
+  diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-hadolint.txt" | ${AWK} '{print $1}')
+
+
+  ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))
+
+  statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}"
)
+
+  if [[ ${diffPostpatch} -gt 0 ]] ; then
+    add_vote_table -1 hadolint "${BUILDMODEMSG} ${statstring}"
+    add_footer_table hadolint "@@BASE@@/diff-patch-hadolint.txt"
+    bugsystem_linecomments "hadolint" "${PATCH_DIR}/diff-patch-hadolint.txt"
+    return 1
+  elif [[ ${fixedpatch} -gt 0 ]]; then
+    add_vote_table +1 hadolint "${BUILDMODEMSG} ${statstring}"
+    return 0
+  fi
+
+  add_vote_table +1 hadolint "There were no new hadolint issues."
+  return 0
+}
+
+function hadolint_postcompile
+{
+  declare repostatus=$1
+
+  if [[ "${repostatus}" = branch ]]; then
+    hadolint_preapply
+  else
+    hadolint_postapply
+  fi
+}


Mime
View raw message