yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [yetus] 03/03: YETUS-738. Some Jenkinsfile improvements
Date Sun, 30 Dec 2018 17:20:45 GMT
This is an automated email from the ASF dual-hosted git repository.

aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git

commit ae6d2fb6257336c3711bab40b3a856d53bb18ca1
Author: Allen Wittenauer <aw@apache.org>
AuthorDate: Tue Dec 18 22:10:16 2018 -0800

    YETUS-738. Some Jenkinsfile improvements
    
    Signed-off-by: Allen Wittenauer <aw@apache.org>
---
 Jenkinsfile | 251 +++++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 197 insertions(+), 54 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 04396eb..0046404 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -15,104 +15,247 @@
 // specific language governing permissions and limitations
 // under the License.
 pipeline {
+
   agent {
     // Hadoop and ubuntu for ASF, rest are private
     label 'Hadoop||ubuntu||azaka||small'
   }
+
+  // Waiting on INFRA-17471 so that webhooks work
+  // in the meantime ...
   triggers {
-    cron('@daily')
+    pollSCM('@hourly')
   }
+
   options {
     buildDiscarder(logRotator(numToKeepStr: '5'))
     timeout (time: 9, unit: 'HOURS')
     timestamps()
     checkoutToSubdirectory('src')
   }
+
   environment {
-    //YETUS_RELEASE = '0.8.0'
     YETUS_BASEDIR = 'src'
-    // will also need to change email section below
+    // will also need to change notification section below
     YETUS_RELATIVE_PATCHDIR = 'out'
     YETUS_DOCKERFILE = "${YETUS_BASEDIR}/precommit/src/main/shell/test-patch-docker/Dockerfile"
   }
+
+  parameters {
+    string(name: 'ISSUE_NUM',
+           defaultValue: '',
+           description: 'The JIRA YETUS issue number that has a patch needing pre-commit
testing. Example: 1234')
+    booleanParam(name: 'USE_DEBUG_FLAG',
+                 defaultValue: false,
+                 description: 'click to enable extra outputs')
+    booleanParam(name: 'USE_DOCKER_FLAG',
+                 defaultValue: true,
+                 description: 'Disable to turn off Docker mode')
+    string(name: 'EXTRA_ARGS',
+           defaultValue: '',
+           description: 'Any extra test-patch arguments')
+  }
+
   stages {
     stage ('precommit-run') {
       steps {
-        sh '''#!/usr/bin/env bash
-            if [[ -d "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}" ]]; then
-              rm -rf "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
-            fi
-            mkdir -p "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+        withCredentials([usernamePassword(credentialsId: 'apache-yetus-at-github.com',
+                         passwordVariable: 'GITHUB_PASSWORD',
+                         usernameVariable: 'GITHUB_USER')]) {
+          withCredentials([usernamePassword(credentialsId: 'yetusqa-at-asf-jira',
+                           passwordVariable: 'JIRA_PASSWORD',
+                           usernameVariable: 'JIRA_USER')]) {
+            sh '''#!/usr/bin/env bash
+
+                # The ASF Jenkins servers are always full of broken JVMs left over
+                # from really terrible jobs/bugs in Java. This should get moved to
+                # the core code as part of YETUS-745
+
+                uptime
+
+                free -h
+
+                pidscur=$(cat /sys/fs/cgroup/pids/user.slice/user-${UID}.slice/pids.current)
+                pidsmax=$(cat /sys/fs/cgroup/pids/user.slice/user-${UID}.slice/pids.max)
+                ((remainingpids=pidsmax - pidscur))
+
+                echo "PIDS:  max: ${pidsmax} cur: ${pidscur} rem: ${remainingpids}"
+
+                # Doing this lets us log them. We pull
+                # out test-patch since --java-home + --jira-home might appear on
+                # its command line
+
+                pids=$(ps -ef | grep /home/jenkins/jenkins-slave/workspace/ | grep -v test-patch
| awk '{print $2}')
+
+
+                for pid in ${pids}; do
+                  elapsed=$(ps -o etimes= -p ${pid})
+                  if [[ ${elapsed} -gt 86400 ]]; then
+                    ps up ${pid}
+                    echo "Killing ${pid} ***"
+                    kill -9 "${pid}"
+                  fi
+                done
+
+                uptime
+
+                free -h
+
+                pidscur=$(cat /sys/fs/cgroup/pids/user.slice/user-${UID}.slice/pids.current)
+                pidsmax=$(cat /sys/fs/cgroup/pids/user.slice/user-${UID}.slice/pids.max)
+                ((remainingpids=pidsmax - pidscur))
 
-            # where the source is located
-            YETUS_ARGS+=("--basedir=${WORKSPACE}/${YETUS_BASEDIR}")
+                echo "PIDS:  max: ${pidsmax} cur: ${pidscur} rem: ${remainingpids}"
 
-            # nuke the src repo before working
-            YETUS_ARGS+=("--resetrepo")
+                # clean and make a new directory for our output artifacts, temporary
+                # storage, etc just incase the workspace directory
+                # delete in post is removed
+                if [[ -d "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}" ]]; then
+                  rm -rf "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+                fi
+                mkdir -p "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+                YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
 
-            # Enable maven custom repos in order to avoid multiple executor clashes
-            YETUS_ARGS+=("--mvn-custom-repos")
+                # where the source is located
+                YETUS_ARGS+=("--basedir=${WORKSPACE}/${YETUS_BASEDIR}")
 
-            # run in docker mode
-            YETUS_ARGS+=("--docker")
+                # our project defaults come from a personality file
+                # which will get loaded automatically by setting the project name
+                YETUS_ARGS+=("--project=yetus")
 
-            # temp storage, etc
-            YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
+                # Enable maven custom repos in order to avoid multiple executor clashes
+                YETUS_ARGS+=("--mvn-custom-repos")
 
-            # lots of different output formats
-            YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt")
-            YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt")
-            YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html")
+                # turn on the sentinel to keep our build systems clean
+                YETUS_ARGS+=(--sentinel)
 
-            # rsync these files back into the archive dir
-            YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+                # lots of different output formats
+                YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt")
+                YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt")
+                YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html")
 
-            # URL for user-side presentation
-            YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+                # enable writing back to Github
+                YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
+                YETUS_ARGS+=(--github-user=${GITHUB_USER})
 
-            # plugins to enable
-            YETUS_ARGS+=("--plugins=all")
+                # enable writing back to ASF JIRA
+                YETUS_ARGS+=(--jira-issue-re='^YETUS-[0-9]*$')
+                YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
+                YETUS_ARGS+=(--jira-user="${JIRA_USER}")
 
-            YETUS_ARGS+=("--tests-filter=checkstyle,javadoc,rubocop,ruby-lint,tests4tests")
+                # disable per-line comments
+                YETUS_ARGS+=(--linecomments='')
 
-            # run test-patch from the source tree specified up above
-            TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/test-patch.sh
+                # auto-kill any surefire stragglers during unit test runs
+                YETUS_ARGS+=(--reapermode=report)
 
-            /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+                # set a super high proclimit
+                YETUS_ARGS+=(--proclimit=2000)
 
-            '''
+                # rsync these files back into the archive dir
+                YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+                # URL for user-side presentation in reports and such to our artifacts
+                # (needs to match the archive bits below)
+                YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+                # plugins to enable
+                YETUS_ARGS+=("--plugins=all")
+
+                # don't let these tests cause -1s because we aren't really paying that
+                # much attention to them
+                YETUS_ARGS+=("--tests-filter=checkstyle,javadoc,rubocop,ruby-lint,test4tests")
+
+                if [[ "${USE_DEBUG_FLAG}" == true ]]; then
+                  YETUS_ARGS+=("--debug")
+                fi
+
+                if [[ -n "${ISSUE_NUM}" ]]; then
+                  YETUS_ARGS+=("YETUS-${ISSUE_NUM}")
+                fi
+
+                # run in docker mode and specifically point to our
+                # Dockerfile since we don't want to use the auto-pulled version.
+                if [[ "${USE_DOCKER_FLAG}" == true ]]; then
+                  YETUS_ARGS+=("--docker")
+                  YETUS_ARGS+=("--dockerfile=${YETUS_DOCKERFILE}")
+                else
+                  # need to figure this out programmatically; hard-coded for now
+                  export JAVA_HOME=/home/jenkins/tools/java/latest1.8
+                  export MAVEN_HOME=/home/jenkins/tools/maven/apache-maven-3.2.1
+                fi
+
+                # run test-patch from the source tree specified up above
+                TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/test-patch.sh
+
+                # execute! (we are using bash instead of the
+                # bin in case the perms get messed up)
+                /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS}
+
+                '''
+          }
+        }
       }
     }
   }
   post {
     always {
-      // Has to be relative to WORKSPACE.
-      archiveArtifacts "${env.YETUS_RELATIVE_PATCHDIR}/**"
-      publishHTML target: [
-        allowMissing: true,
-        keepAll: true,
-        alwaysLinkToLastBuild: true,
-        // Has to be relative to WORKSPACE
-        reportDir: "${env.YETUS_RELATIVE_PATCHDIR}",
-        reportFiles: 'report.html',
-        reportName: 'Yetus QBT Report'
-      ]
+      script {
+        // Publish the HTML report so that it can be looked at
+        // Has to be relative to WORKSPACE.
+        archiveArtifacts "${env.YETUS_RELATIVE_PATCHDIR}/**"
+        publishHTML (target: [
+                      allowMissing: true,
+                      keepAll: true,
+                      alwaysLinkToLastBuild: true,
+                      // Has to be relative to WORKSPACE
+                      reportDir: "${env.YETUS_RELATIVE_PATCHDIR}",
+                      reportFiles: 'report.html',
+                      reportName: 'Yetus Report'
+        ])
+          if (env.BRANCH_NAME == 'master') {
+            emailext(subject: '$DEFAULT_SUBJECT',
+                  body:
+'''For more details, see ${BUILD_URL}
+
+${CHANGES, format="[%d] (%a) %m"}
+
+HTML Version: ${BUILD_URL}Yetus_20Report/
+
+
+${FILE,path="out/brief.txt"}
+
+''',
+                  replyTo: 'dev@yetus.apache.org',
+                  to: 'dev@yetus.apache.org'
+          )
+        }
+      }
     }
+
+    // on failure, we send an email to the person who changed
+    // the code and the person who requested the job to get run
     failure {
-      emailext subject: '$DEFAULT_SUBJECT',
-      body: '''For more details, see ${BUILD_URL}
+      emailext(subject: '$DEFAULT_SUBJECT',
+              body:
+'''For more details, see ${BUILD_URL}
 
 ${CHANGES, format="[%d] (%a) %m"}
 
-${FILE,path="out/brief.txt"}''',
-      recipientProviders: [
-          [$class: 'CulpritsRecipientProvider'],
-          [$class: 'DevelopersRecipientProvider'],
-          [$class: 'RequesterRecipientProvider']
-      ],
-      replyTo: '$DEFAULT_REPLYTO',
-      to: '$DEFAULT_RECIPIENTS'
+HTML Version: ${BUILD_URL}Yetus_20Report/
+
+${FILE,path="out/brief.txt"}
+''',
+              recipientProviders: [
+                [$class: 'DevelopersRecipientProvider'],
+                [$class: 'RequesterRecipientProvider']
+              ],
+              replyTo: '$DEFAULT_REPLYTO',
+              to: '$DEFAULT_RECIPIENTS'
+      )
     }
+
+    // Jenkins pipeline jobs fill slaves on PRs without this :(
     cleanup() {
       deleteDir()
     }


Mime
View raw message