hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject hbase git commit: HBASE-19189 Ad-hoc test job for running a subset of tests lots of times [Forced Update!]
Date Tue, 07 Nov 2017 02:44:08 GMT
Repository: hbase
Updated Branches:
  refs/heads/HBASE-19189 4e3a4755c -> 1b1ba46fb (forced update)


HBASE-19189 Ad-hoc test job for running a subset of tests lots of times


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1b1ba46f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1b1ba46f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1b1ba46f

Branch: refs/heads/HBASE-19189
Commit: 1b1ba46fb91496e75de12c6c41c490d9de071a42
Parents: 28cdf4a
Author: Sean Busbey <busbey@apache.org>
Authored: Mon Nov 6 13:48:05 2017 -0600
Committer: Sean Busbey <busbey@apache.org>
Committed: Mon Nov 6 20:43:49 2017 -0600

----------------------------------------------------------------------
 dev-support/adhoc_run_tests/Jenkinsfile        | 78 +++++++++++++++++++
 dev-support/adhoc_run_tests/adhoc_run_tests.sh | 83 +++++++++++++++++++++
 dev-support/gather_machine_environment.sh      | 50 +++++++++++++
 3 files changed, 211 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/1b1ba46f/dev-support/adhoc_run_tests/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/adhoc_run_tests/Jenkinsfile b/dev-support/adhoc_run_tests/Jenkinsfile
new file mode 100644
index 0000000..6646bdf
--- /dev/null
+++ b/dev-support/adhoc_run_tests/Jenkinsfile
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+  parameters {
+    string(name: 'tests', description: 'space separated list of tests to run.')
+    string(name: 'node', defaultValue: 'Hadoop', description: 'the node label that should
be used to run the test.')
+  }
+  agent {
+    node {
+      label "${params.node}"
+    }
+  }
+  options {
+    timeout (time: 6, unit: 'HOURS')
+    timestamps()
+  }
+  environment {
+    // where we check out to across stages
+    BASEDIR = "${env.WORKSPACE}/component"
+    OUTPUT_RELATIVE = 'output'
+    OUTPUTDIR = "${env.WORKSPACE}/output"
+    BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+  }
+  stages {
+    stage ('run tests') {
+      tools {
+        maven 'Maven (latest)'
+        // this needs to be set to the jdk that ought to be used to build releases on the
branch the Jenkinsfile is stored in.
+        jdk "JDK 1.8 (latest)"
+      }
+      steps {
+        sh """#!/bin/bash -e
+          echo "Setting up directories"
+          rm -rf "${env.OUTPUTDIR}" && mkdir "${env.OUTPUTDIR}"
+          rm -rf ".m2-repo" && mkdir ".m2-repo"
+          mkdir "${env.OUTPUTDIR}/machine"
+"""
+        sh """#!/bin/bash -e
+          "${env.BASEDIR}/dev-support/gather_machine_environment.sh" \
+              "${OUTPUT_RELATIVE}/machine"
+"""
+        dir ("component") {
+          sh '''#!/bin/bash -e
+            ./dev-support/adhoc_run_tests/adhoc_run_tests.sh \
+                --force-timeout 1800 \
+                --maven-local-repo ".m2-repo" \
+                --log-output "${OUTPUTDIR}" \
+                --repeat 100 \
+                "${tests}"
+'''
+        }
+      }
+      post {
+        always {
+          archive 'output/*'
+          archive 'output/**/*'
+        }
+        failure {
+          archive 'component/**/target/surefire-reports/*'
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/1b1ba46f/dev-support/adhoc_run_tests/adhoc_run_tests.sh
----------------------------------------------------------------------
diff --git a/dev-support/adhoc_run_tests/adhoc_run_tests.sh b/dev-support/adhoc_run_tests/adhoc_run_tests.sh
new file mode 100755
index 0000000..3e61017
--- /dev/null
+++ b/dev-support/adhoc_run_tests/adhoc_run_tests.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+  echo "Usage: ${0} [options] TestSomeTestName [TestOtherTest...]"
+  echo ""
+  echo "    --force-timeout seconds               Seconds to wait before killing a given
test run."
+  echo "    --maven-local-repo /path/to/use       Path for maven artifacts while building"
+  echo "    --repeat times                        number of times to repeat if successful"
+  echo "    --log-output /path/to/use             path to directory to hold attempt log"
+  exit 1
+}
+# Get arguments
+declare -i force_timeout=7200
+declare -i attempts=1
+declare maven_repo="${HOME}/.m2/repository"
+declare output="."
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    --force-timeout) shift; force_timeout=$1; shift;;
+    --maven-local-repo) shift; maven_repo=$1; shift;;
+    --repeat) shift; attempts=$1; shift;;
+    --log-output) shift; output=$1; shift;;
+    --) shift; break;;
+    -*) usage ;;
+    *)  break;;  # terminate while loop
+  esac
+done
+
+if [ "$#" -lt 1 ]; then
+  usage
+fi
+
+function find_module
+{
+  declare testname=$1
+  declare path
+  path=$(find . -name "${testname}.java")
+  while [ -n "${path}" ]; do
+    path=$(dirname "${path}")
+    if [ -f "${path}/pom.xml" ]; then
+      basename "${path}"
+      break
+    fi
+  done
+}
+
+declare -a modules
+
+for test in "${@}"; do
+  module=$(find_module "${test}")
+  if [[ ! "${modules[*]}" =~ ${module} ]]; then
+    modules+=(${module})
+  fi
+done
+
+declare -a mvn_module_arg
+
+for module in "${modules[@]}"; do
+  mvn_module_arg+=(-pl "${module}")
+done
+declare tests="${*}"
+for attempt in $(seq "${attempts}"); do
+  echo "Attempt ${attempt}" >&2
+  mvn --batch-mode -Dmaven.repo.local="${maven_repo}" -Dtest="${tests// /,}" -Dsurefire.rerunFailingTestsCount=0
-Dsurefire.parallel.forcedTimeout="${force_timeout}" -Dsurefire.shutdown=kill -DtrimStackTrace=false
-am "${mvn_module_arg[@]}" package >"${output}/mvn_test.log" 2>&1
+done

http://git-wip-us.apache.org/repos/asf/hbase/blob/1b1ba46f/dev-support/gather_machine_environment.sh
----------------------------------------------------------------------
diff --git a/dev-support/gather_machine_environment.sh b/dev-support/gather_machine_environment.sh
new file mode 100755
index 0000000..8f2c74e
--- /dev/null
+++ b/dev-support/gather_machine_environment.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+  echo "Usage: ${0} /path/for/output/dir"
+  echo ""
+  echo "  Gather info about a build machine that test harnesses should poll before running."
+  echo "  presumes you'll then archive the passed output dir."
+
+  exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+  usage
+fi
+
+
+declare output=$1
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version  >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -l >"${output}/ulimit-l" 2>&1 || true


Mime
View raw message