yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [yetus] branch master updated: YETUS-791. Add support for --cache-from to speed up building
Date Tue, 05 Feb 2019 23:30:36 GMT
This is an automated email from the ASF dual-hosted git repository.

aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git


The following commit(s) were added to refs/heads/master by this push:
     new 786d288  YETUS-791. Add support for --cache-from to speed up building
786d288 is described below

commit 786d2886f0b52d3a6fc01b4fa30dcc58baecf43d
Author: Allen Wittenauer <aw@apache.org>
AuthorDate: Sat Feb 2 09:56:55 2019 -0800

    YETUS-791. Add support for --cache-from to speed up building
    
    Signed-off-by: Allen Wittenauer <aw@apache.org>
---
 Jenkinsfile                                               |  2 ++
 .../source/documentation/in-progress/precommit-docker.md  |  4 ++++
 precommit/src/main/shell/core.d/docker.sh                 | 15 +++++++++++++++
 start-build-env.sh                                        |  7 +++++--
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e067994..ff9a624 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -172,8 +172,10 @@ pipeline {
                 # run in docker mode and specifically point to our
                 # Dockerfile since we don't want to use the auto-pulled version.
                 if [[ "${USE_DOCKER_FLAG}" == true ]]; then
+                  docker pull ubuntu:xenial
                   YETUS_ARGS+=("--docker")
                   YETUS_ARGS+=("--dockerfile=${YETUS_DOCKERFILE}")
+                  YETUS_ARGS+=("--docker-cache-from=apache/yetus-base:master")
                 else
                   # need to figure this out programmatically; hard-coded for now
                   export JAVA_HOME=/home/jenkins/tools/java/latest1.8
diff --git a/asf-site-src/source/documentation/in-progress/precommit-docker.md b/asf-site-src/source/documentation/in-progress/precommit-docker.md
index f65d1bf..c051a1c 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-docker.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-docker.md
@@ -57,6 +57,10 @@ In order to use both 'YETUS CUT HERE' and a Dockerfile that uses COPY and
ADD di
 
 Instead of processing a Dockerfile, test-patch can pull a tag from a repository using the
`--docker-tag` parameter. Note that the repository must already be logged into and configured
prior to executing test-patch.
 
+## Using a cache
+
+With the `--docker-cache-from` parameter, other images may be utilized to provide a cache
when building a Dockerfile. This comma delimited list will automatically be pulled (errors
are ignored) and given to the docker command line to use.
+
 ## Platforms
 
 When either building or pull an image, test-patch supports the `--docker-platform` flag to
pass in the Docker `--platform` flag.  This allows you full control over what kind of image
the software either creates or fetches.
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index d298d19..12621d3 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -62,6 +62,7 @@ function docker_usage
   yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default:
'${DOCKERCMD}')"
   if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
     yetus_add_option "--docker-bash-debug=<bool>" "Enable bash -x mode running in a
container (default: ${YETUS_DOCKER_BASH_DEBUG})"
+    yetus_add_option "--docker-cache-from=<image>" "Comma delimited images to use as
a cache when building"
     yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base
(default: '${DOCKERFILE_DEFAULT}')"
     yetus_add_option "--dockerind=<bool>" "Enable Docker-in-Docker by mounting the
Docker socket in the container (default: '${DOCKER_IN_DOCKER}')"
     yetus_add_option "--docker-platform=<plat>" "Use a platform string for building
and pulling (default: ${DOCKER_PLATFORM})"
@@ -93,6 +94,9 @@ function docker_parse_args
         YETUS_DOCKER_BASH_DEBUG=${i#*=}
         add_docker_env YETUS_DOCKER_BASH_DEBUG
       ;;
+      --docker-cache-from=*)
+        DOCKER_CACHE_FROM=${i#*=}
+      ;;
       --dockercmd=*)
         #shellcheck disable=SC2034
         DOCKERCMD=${i#*=}
@@ -561,6 +565,8 @@ function docker_run_image
   declare lines
   declare dockerversion
   declare -a dockplat
+  declare -a cachefrom
+  declare -a images
 
   big_console_header "Docker Image Creation"
   start_clock
@@ -633,8 +639,17 @@ function docker_run_image
       cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile"
     fi
 
+    if [[ -n "${DOCKER_CACHE_FROM}" ]]; then
+      yetus_comma_to_array images "${DOCKER_CACHE_FROM}"
+      for i in "${images[@]}"; do
+        docker pull "${i}" || true
+      done
+      cachefrom=("--cache-from=yetus/${PROJECT_NAME}:${gitfilerev},${DOCKER_CACHE_FROM}")
+    fi
+
     if ! dockercmd build \
           "${dockplat[@]}" \
+          "${cachefrom[@]}" \
           --label org.apache.yetus=\"\" \
           --label org.apache.yetus.testpatch.project="${PROJECT_NAME}" \
           --tag "${baseimagename}" \
diff --git a/start-build-env.sh b/start-build-env.sh
index d2d024a..dc7160d 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -26,11 +26,14 @@ if [[ "${BRANCH}" =~ HEAD ]]; then
   BRANCH=$(git branch | grep '\*' | awk '{print $NF}'  | sed -e s,rel/,,g -e s,\),,g )
 fi
 
-echo "Attempting a pull of apache/yetus-base:${BRANCH} and apache/yetus-base:master to save
time"
+echo "Attempting a few pulls of apache/yetus and apache/yetus-base to save time"
 echo "Errors here will be ignored!"
 docker pull "apache/yetus-base:${BRANCH}" || docker pull "apache/yetus-base:master" || true
+docker pull "apache/yetus:${BRANCH}"  || docker pull "apache/yetus:master" || true
 
-docker build -t "apache/yetus-build:${BRANCH}" .
+docker build \
+  --cache-from="apache/yetus-base:${BRANCH},apache/yetus-base:master,apache/yetus:${BRANCH},apache/yetus:master"
\
+  -t "apache/yetus-build:${BRANCH}" .
 
 USER_NAME=${SUDO_USER:=$USER}
 USER_ID=$(id -u "${USER_NAME}")


Mime
View raw message