Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 62B1E186C0 for ; Tue, 10 Nov 2015 19:05:45 +0000 (UTC) Received: (qmail 78858 invoked by uid 500); 10 Nov 2015 19:05:45 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 78803 invoked by uid 500); 10 Nov 2015 19:05:45 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 78195 invoked by uid 99); 10 Nov 2015 19:05:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Nov 2015 19:05:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 34DEDDFF8D; Tue, 10 Nov 2015 19:05:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: syuanjiang@apache.org To: commits@hbase.apache.org Date: Tue, 10 Nov 2015 19:05:52 -0000 Message-Id: <52c0643628bf4d1399c200271b82aa90@git.apache.org> In-Reply-To: <474459395d44492181dab069c54fa644@git.apache.org> References: <474459395d44492181dab069c54fa644@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [09/22] hbase git commit: HBASE-14772 Improve zombie detector; be more discerning HBASE-14772 Improve zombie detector; be more discerning Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bea2f7fe Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bea2f7fe Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bea2f7fe Branch: refs/heads/hbase-12439 Commit: bea2f7feacd1a34d27ee17c201aaeacc32e8cdaf Parents: 0896318 Author: stack Authored: Fri Nov 6 16:39:15 2015 -1000 Committer: stack Committed: Fri Nov 6 16:39:15 2015 -1000 ---------------------------------------------------------------------- dev-support/zombie-detector.sh | 151 ++++++++++++++++++++++++++++++++++++ pom.xml | 11 ++- 2 files changed, 159 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/bea2f7fe/dev-support/zombie-detector.sh ---------------------------------------------------------------------- diff --git a/dev-support/zombie-detector.sh b/dev-support/zombie-detector.sh new file mode 100644 index 0000000..57c1374 --- /dev/null +++ b/dev-support/zombie-detector.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Looks for any running zombies left over from old build runs. +# Will report and try to do stack trace on stale processes so can +# figure how they are hung. + +# TODO: format output to suit context -- test-patch, jenkins or dev env + +#set -x +# REMOVE +printenv + +### Setup some variables. +bindir=$(dirname $0) + +# This key is set by our surefire configuration up in the main pom.xml +# This key needs to match the key we set up there. +HBASE_BUILD_ID_KEY="hbase.build.id=" +JENKINS=false + +PS=${PS:-ps} +AWK=${AWK:-awk} +WGET=${WGET:-wget} +GREP=${GREP:-grep} +JIRACLI=${JIRA:-jira} + +############################################################################### +printUsage() { + echo "Usage: $0 [options]" BUILD_ID + echo + echo "Where:" + echo " BUILD_ID is build id to look for in process listing" + echo + echo "Options:" + echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--awk-cmd= The 'awk' command to use (default 'awk')" + echo "--grep-cmd= The 'grep' command to use (default 'grep')" + echo + echo "Jenkins-only options:" + echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--wget-cmd= The 'wget' command to use (default 'wget')" + echo "--jira-cmd= The 'jira' command to use (default 'jira')" +} + +############################################################################### +parseArgs() { + for i in $* + do + case $i in + --jenkins) + JENKINS=true + ;; + --ps-cmd=*) + PS=${i#*=} + ;; + --awk-cmd=*) + AWK=${i#*=} + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --jira-cmd=*) + JIRACLI=${i#*=} + ;; + *) + BUILD_ID=$i + ;; + esac + done + if [ -z "$BUILD_ID" ]; then + printUsage + exit 1 + fi + if [[ $JENKINS == "true" ]] ; then + echo "Running in Jenkins mode" + else + echo "Running in developer mode" + JENKINS=false + fi +} + +### Return list of the processes found with passed build id. +find_processes () { + jps -v | grep surefirebooter | grep -e "${HBASE_BUILD_TAG}" +} + +### Look for zombies +zombies () { + ZOMBIES=`find_processes` + # xargs trims white space before and after the count + ZOMBIE_TESTS_COUNT=`echo "${ZOMBIES}"|wc -l|xargs` + if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then + wait=15 + echo "Found ${ZOMBIE_TESTS_COUNT} suspicious java process(es); waiting ${wait}s to see if just slow to stop" + sleep ${wait} + PIDS=`echo "${ZOMBIES}"|${AWK} '{print $1}'` + ZOMBIE_TESTS_COUNT=0 + for pid in $PIDS + do + # Test our zombie still running (and that it still an hbase build item) + PS_OUTPUT=`ps -p $pid | tail +2 | grep -e "${HBASE_BUILD_TAG}"` + if [[ ! -z "${PS_OUTPUT}" ]] + then + echo "Zombie: $PS_OUTPUT" + let "ZOMBIE_TESTS_COUNT+=1" + PS_STACK=`jstack $pid | grep -e "\.Test" | grep -e "\.java"| head -3` + echo "${PS_STACK}" + ZB_STACK="${ZB_STACK}\npid=${pid} ${PS_STACK}" + fi + done + if [[ $ZOMBIE_TESTS_COUNT != 0 ]] + then + # If JIRA_COMMENT in environment, append our findings to it + JIRA_COMMENT="$JIRA_COMMENT + {color:red}-1 core zombie tests{color}. There are ${ZOMBIE_TESTS_COUNT} possible zombie test(s): ${ZB_STACK}" + # Exit with error + exit 1 + fi + fi +} + +### Check if arguments to the script have been specified properly or not +parseArgs $@ +HBASE_BUILD_TAG="${HBASE_BUILD_ID_KEY}${BUILD_ID}" +zombies +RESULT=$? +if [[ $JENKINS == "true" ]] ; then + if [[ $RESULT != 0 ]] ; then + exit 100 + fi +fi +RESULT=$? http://git-wip-us.apache.org/repos/asf/hbase/blob/bea2f7fe/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2d35610..b9505e6 100644 --- a/pom.xml +++ b/pom.xml @@ -1236,9 +1236,9 @@ 2800m 2800m - - -enableassertions -Dhbase.test -Xmx${surefire.Xmx} + -enableassertions -Dhbase.build.id=${build.id} -Xmx${surefire.Xmx} -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true @@ -1252,7 +1252,12 @@ 1.0-beta-3 ${project.build.directory}/test-classes - + yyyy-MM-dd'T'HH:mm:ss'Z' + + ${maven.build.timestamp} +