yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [yetus] 01/04: YETUS-764. test-patch running in a pipeline isn't getting killed
Date Wed, 09 Jan 2019 23:39:34 GMT
This is an automated email from the ASF dual-hosted git repository.

aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git

commit 8f31a4c8c88a314e7c76fa212a9abb75a1c98c94
Author: Allen Wittenauer <aw@apache.org>
AuthorDate: Sun Jan 6 10:42:49 2019 -0800

    YETUS-764. test-patch running in a pipeline isn't getting killed
    
    Signed-off-by: Sean Busbey <busbey@apache.org>
---
 Jenkinsfile                                        | 22 ++++++++++++++++--
 .../documentation/in-progress/precommit-robots.md  | 27 ++++++++++++++++++++--
 precommit/src/main/shell/core.d/docker.sh          | 12 ++++++----
 precommit/src/main/shell/test-patch.sh             |  8 +++++++
 4 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 78a623a..aee1009 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,9 +254,27 @@ ${FILE,path="out/brief.txt"}
       }
     }
 
-    // Jenkins pipeline jobs fill slaves on PRs without this :(
+    // Jenkins has issues in some configurations sending
+    // signals to child processes. Additionally, Github Branch
+    // Source plug-in will quickly fill Jenkins build hosts
+    // with PR directories.  This cleanup stanze kills
+    // any left over processes/containers and frees disk space
+    // on exit
     cleanup() {
-      deleteDir()
+      script {
+        sh '''
+            if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
+              echo "test-patch process appears to still be running: killing"
+              kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
+              sleep 10
+            fi
+            if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
+              echo "test-patch container appears to still be running: killing"
+              docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
+            fi
+            '''
+        deleteDir()
+      }
     }
   }
 }
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
index faa487c..9c226ac 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-robots.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -113,12 +113,35 @@ pipeline {
     JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
   }
 
-  ...
-
 }
 
 ```
 
+ Experience has shown that certain Jenkins + Java + OS combinations have problems sending
signals to child processes.  In the case of Apache Yetus, this may result in aborted or workflows
that timeout not being properly killed.  `test-patch` will write two files in the patch directory
that may be helpful to combat this situation if it applies to your particular configuration.
 `pidfile.txt` contains the master `test-patch` process id and `cidfile.txt` contains the
docker container id.  T [...]
+
+ ```groovy
+    post {
+      cleanup() {
+        script {
+          sh '''
+            if [ -f "${env.PATCH_DIR}/pidfile.txt" ]; then
+              kill `cat "${env.PATCH_DIR}/pidfile.txt"` || true
+              sleep 5
+            fi
+            if [ -f "${env.PATCH_DIR}/cidfile.txt" ]; then
+              docker kill `cat "${env.PATCH_DIR}/cidfile.txt"` || true
+              sleep 5
+            fi
+            '''
+            ...
+            deletedir()
+        }
+      }
+    }
+ ```
+
+
+
 See also
   * See also the source tree's `Jenkinsfile` for some tips and tricks.
   * [precommit-admin](precommit-admin), for special utilities built for Jenkins.
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index d419bc0..d298d19 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -717,7 +717,7 @@ function docker_run_image
   # make the kernel prefer to kill us if we run out of RAM
   DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
 
-  DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile")
+  DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile.txt")
 
   if [[ "${DOCKER_IN_DOCKER}" == true ]]; then
     if [[ -e "${DOCKER_SOCKET}" ]]; then
@@ -735,8 +735,7 @@ function docker_run_image
 
   DOCKER_EXTRAARGS+=(--name "${containername}")
 
-  trap 'docker_signal_handler' SIGTERM
-  trap 'docker_signal_handler' SIGINT
+  trap 'docker_signal_handler' SIGTERM SIGINT SIGHUP
 
   if [[ ${PATCH_DIR} =~ ^/ ]]; then
     dockercmd run --rm=true -i \
@@ -758,6 +757,7 @@ function docker_run_image
   printf '\n\n'
   echo "Cleaning up docker image used for testing."
   dockercmd rmi "${patchimagename}" > /dev/null
+  rm "${PATCH_DIR}/cidfile.txt"
   cleanup_and_exit ${retval}
 }
 
@@ -769,10 +769,12 @@ function docker_signal_handler
 {
   declare cid
 
-  cid=$(cat "${PATCH_DIR}/cidfile")
+  cid=$(cat "${PATCH_DIR}/cidfile.txt")
 
   yetus_error "ERROR: Caught signal. Killing docker container:"
-  dockercmd kill "${cid}"
+  echo "ERROR: Caught signal. Killing docker container: ${cid}" > "${PATCH_DIR}/signal.log"
+  dockercmd kill "${cid}" | tee -a "${PATCH_DIR}/signal.log"
+  rm "${PATCH_DIR}/cidfile.txt"
   yetus_error "ERROR: Exiting."
   cleanup_and_exit 143 # 128 + 15 -- SIGTERM
 }
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index 4ea5cb1..b361289 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -2470,6 +2470,10 @@ function cleanup_and_exit
     big_console_header "Finished build."
   fi
 
+  if [[ "${DOCKERMODE}" != true ]]; then
+    rm "${PATCH_DIR}/pidfile.txt"
+  fi
+
   # shellcheck disable=SC2086
   exit ${result}
 }
@@ -3210,6 +3214,10 @@ function initialize
     docker_initialize
   fi
 
+  if [[ "${DOCKERMODE}" != true ]]; then
+    echo "$$" > "${PATCH_DIR}/pidfile.txt"
+  fi
+
   plugins_initialize
   if [[ ${RESULT} != 0 ]]; then
     cleanup_and_exit 1


Mime
View raw message