Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 18F4910C8E for ; Mon, 6 Jan 2014 19:09:48 +0000 (UTC) Received: (qmail 71958 invoked by uid 500); 6 Jan 2014 19:09:47 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 71775 invoked by uid 500); 6 Jan 2014 19:09:46 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 71392 invoked by uid 99); 6 Jan 2014 19:09:46 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Jan 2014 19:09:46 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1CC64882ACD; Mon, 6 Jan 2014 19:09:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bhavanki@apache.org To: commits@accumulo.apache.org Date: Mon, 06 Jan 2014 19:09:50 -0000 Message-Id: <83084417f63a403db2da8e292c9306e1@git.apache.org> In-Reply-To: <2f008ffdf1664569b5d6bb9a9c5aa578@git.apache.org> References: <2f008ffdf1664569b5d6bb9a9c5aa578@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [07/20] git commit: ACCUMULO-1944 Fix coverage for functional tests ACCUMULO-1944 Fix coverage for functional tests The -C flag for test/system/auto/run.py did not work. This changeset allows it to work again if a functional Cobertura installation is placed under $ACCUMULO_HOME/lib/test/cobertura. The code for producing instrumented Accumulo code was inactive and out of date. It was reworked so that, if -C is passed, the Accumulo JARs are instrumented and placed into a location ahead of their standard location in the test classpath. (If -C is not passed, any instrumented JARs are removed.) The classpath is also dynamically adjusted to include whatever Cobertura JAR is available; its name includes a version number (as of 2.0.x). The command-line scripts shipped with Cobertura 2.0.x are out of date and do not work out of the box. Pull request #102 was submitted to cobertura/cobertura on Github to fix the problem; in lieu of that, the scripts must be manually updated to use a correct classpath. Tested with Cobertura 2.0.3 / 2.0.4-SNAPSHOT with needed script updates. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/198b1abd Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/198b1abd Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/198b1abd Branch: refs/heads/master Commit: 198b1abd9e81d1af55220bf22a0b4e6ec1d18048 Parents: 25a7fda Author: Bill Havanki Authored: Mon Dec 2 15:33:56 2013 -0500 Committer: Bill Havanki Committed: Mon Jan 6 13:20:29 2014 -0500 ---------------------------------------------------------------------- test/system/auto/TestUtils.py | 20 +++++++++++-- test/system/auto/run.py | 57 ++++++++++++++++++++++++++------------ 2 files changed, 58 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/198b1abd/test/system/auto/TestUtils.py ---------------------------------------------------------------------- diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py old mode 100755 new mode 100644 index 3101cf9..63a45b2 --- a/test/system/auto/TestUtils.py +++ b/test/system/auto/TestUtils.py @@ -24,6 +24,7 @@ import socket import signal import select import random +import re import shutil import sleep @@ -49,6 +50,13 @@ else: SITE = "test-" + ID SITE_PATH = os.path.join(ACCUMULO_CONF_DIR, SITE) +COBERTURA_HOME = os.path.join(ACCUMULO_HOME, 'lib', 'test', 'cobertura') +def findCoberturaJar(): + jars = [f for f in os.listdir(COBERTURA_HOME) if re.search(r'cobertura.*\.jar', f)] + if len(jars) >= 1: + return jars[0] + return None + WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID) LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties') @@ -63,6 +71,8 @@ $ACCUMULO_HOME/server/target/classes/, $ACCUMULO_HOME/fate/target/classes/, $ACCUMULO_HOME/examples/instamo/target/classes, $ACCUMULO_HOME/examples/simple/target/classes, + $ACCUMULO_HOME/instrumented/[^.].*.jar, + $ACCUMULO_HOME/lib/test/cobertura/cobertura.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar, $HADOOP_CONF_DIR, @@ -262,8 +272,14 @@ class TestUtilsMixin: self.pkill(host, 'org.apache.accumulo.start', signal) def create_config_file(self, settings): + cobertura_jar = findCoberturaJar() + if cobertura_jar: + settings_classpath = General_CLASSPATH.replace('cobertura.jar', os.path.basename(cobertura_jar)) + else: + settings_classpath = General_CLASSPATH + fp = open(SITE_PATH, 'w') - fp.write('\n') + fp.write('\n') settings = self.settings.copy() settings.update({ 'instance.zookeeper.host': ZOOKEEPERS, 'instance.dfs.dir': ACCUMULO_DIR, @@ -273,7 +289,7 @@ class TestUtilsMixin: 'logger.port.client': 44000 + FUZZ, 'gc.port.client': 45000 + FUZZ, 'logger.dir.walog': WALOG, - 'general.classpaths' :General_CLASSPATH, + 'general.classpaths': settings_classpath, 'instance.secret': 'secret', }) for a, v in settings.items(): http://git-wip-us.apache.org/repos/asf/accumulo/blob/198b1abd/test/system/auto/run.py ---------------------------------------------------------------------- diff --git a/test/system/auto/run.py b/test/system/auto/run.py index 8e1a9bb..9db1f64 100755 --- a/test/system/auto/run.py +++ b/test/system/auto/run.py @@ -22,11 +22,11 @@ import logging import unittest import glob import re +import shutil import sys from subprocess import Popen, PIPE -from TestUtils import ACCUMULO_HOME, ACCUMULO_DIR -COBERTURA_HOME = os.path.join(ACCUMULO_HOME, 'lib', 'test', 'cobertura') +from TestUtils import ACCUMULO_HOME, ACCUMULO_DIR, COBERTURA_HOME, findCoberturaJar import sleep log = logging.getLogger('test.auto') @@ -108,26 +108,39 @@ def removeCoverageFromPreviousRun(): except OSError: pass +def classpath(dir): + return ':'.join([f for f in os.listdir(dir) if f.endswith('.jar')]) + def instrumentAccumuloJar(jar): - instrumented = jar[:-4] + "-instrumented" + ".jar" - try: - os.unlink(instrumented) - except OSError: - pass - os.link(jar, instrumented) + basedir = os.path.join(ACCUMULO_HOME, 'lib') + instpath = os.path.join(ACCUMULO_HOME, 'instrumented') + if not os.access(instpath, os.F_OK): + os.mkdir(instpath) + auxcp = classpath(basedir) + jarname = os.path.basename(jar) + destjar = os.path.join(instpath, jarname) + if os.access(destjar, os.F_OK): + print "%s already instrumented" % jarname + return destjar cmd = os.path.join(COBERTURA_HOME, "cobertura-instrument.sh") - run(['sh', '-c', '%s --includeClasses "accumulo.*" %s' % ( - cmd, instrumented)]) + print '- Instrumenting %s' % jarname + run(['sh', '-c', '%s --basedir %s --destination %s --auxClasspath %s %s' % (cmd, basedir, instpath, auxcp, jarname)]) assert os.path.exists('cobertura.ser') - return instrumented + return destjar + +def removeInstrumentedAccumuloJars(): + instpath = os.path.join(ACCUMULO_HOME, 'instrumented') + shutil.rmtree(instpath, ignore_errors=True) def mergeCoverage(): "Most of the coverage ends up in $HOME due to ssh'ing around" fname = 'cobertura.ser' - run(['sh', '-c', ' '.join([ - os.path.join(COBERTURA_HOME, "cobertura-merge.sh"), - os.path.join(os.environ['HOME'], fname), - fname])]) + if os.access(os.path.join(os.environ['HOME'], fname), os.F_OK): + run(['sh', '-c', ' '.join([ + os.path.join(COBERTURA_HOME, "cobertura-merge.sh"), + os.path.join(os.environ['HOME'], fname), + fname])]) + def produceCoverageReport(sourceDirectories): reporter = os.path.join(COBERTURA_HOME, 'cobertura-report.sh') @@ -284,11 +297,21 @@ def main(): map(suite.addTest, filtered) if options.coverage: + cobertura_jar = os.path.join(COBERTURA_HOME, findCoberturaJar()) + if not cobertura_jar or not os.access(cobertura_jar, os.F_OK): + print "Install Cobertura under %s" % COBERTURA_HOME + sys.exit(1) fixCoberturaShellScripts() removeCoverageFromPreviousRun() - os.environ['HADOOP_CLASSPATH'] = os.path.join(COBERTURA_HOME, - 'cobertura.jar') + os.environ['HADOOP_CLASSPATH'] = cobertura_jar sleep.scale = 2.0 + libpath = os.path.join(ACCUMULO_HOME,'lib') + libs = os.listdir(libpath) + for l in libs: + if re.search(r'^accumulo.*\.jar$', l): + instrumentAccumuloJar(os.path.join(libpath, l)) + else: + removeInstrumentedAccumuloJars() for i in range(options.repeat): runner.run(suite)