Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1745B10FC1 for ; Fri, 27 Sep 2013 15:01:06 +0000 (UTC) Received: (qmail 21779 invoked by uid 500); 27 Sep 2013 15:01:05 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 19440 invoked by uid 500); 27 Sep 2013 15:00:59 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 19432 invoked by uid 99); 27 Sep 2013 15:00:58 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Sep 2013 15:00:58 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Sep 2013 15:00:53 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0814A2388C4E; Fri, 27 Sep 2013 15:00:32 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1526925 - in /hive/trunk/testutils/ptest2/src: main/java/org/apache/hive/ptest/execution/ main/java/org/apache/hive/ptest/execution/conf/ main/java/org/apache/hive/ptest/execution/context/ main/resources/ test/java/org/apache/hive/ptest/ex... Date: Fri, 27 Sep 2013 15:00:31 -0000 To: commits@hive.apache.org From: brock@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130927150032.0814A2388C4E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: brock Date: Fri Sep 27 15:00:31 2013 New Revision: 1526925 URL: http://svn.apache.org/r1526925 Log: HIVE-5361 - PTest2 should allow a different JVM for compilation versus execution Added: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt Removed: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestCleanupPhase.java Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original) +++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Fri Sep 27 15:00:31 2013 @@ -110,6 +110,7 @@ public class PTest { put("buildTag", buildTag). put("logDir", logDir.getAbsolutePath()). put("javaHome", configuration.getJavaHome()). + put("javaHomeForTests", configuration.getJavaHomeForTests()). put("antEnvOpts", configuration.getAntEnvOpts()); final ImmutableMap templateDefaults = templateDefaultsBuilder.build(); TestParser testParser = new TestParser(configuration.getContext(), @@ -221,12 +222,13 @@ public class PTest { } private static final String PROPERTIES = "properties"; - private static final String REPOSITORY = "repository"; - private static final String REPOSITORY_NAME = "repositoryName"; - private static final String BRANCH = "branch"; + private static final String REPOSITORY = TestConfiguration.REPOSITORY; + private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME; + private static final String BRANCH = TestConfiguration.BRANCH; private static final String PATCH = "patch"; - private static final String JAVA_HOME = "javaHome"; - private static final String ANT_ENV_OPTS = "antEnvOpts"; + private static final String JAVA_HOME = TestConfiguration.JAVA_HOME; + private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST; + private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS; /** * All args override properties file settings except * for this one which is additive. @@ -243,7 +245,8 @@ public class PTest { options.addOption(null, BRANCH, true, "Overrides git branch in properties file"); options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://"); options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments"); - options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests"); + options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)"); + options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)"); options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environemnt variable setting"); CommandLine commandLine = parser.parse(options, args); if(!commandLine.hasOption(PROPERTIES)) { @@ -282,6 +285,10 @@ public class PTest { if(!javaHome.isEmpty()) { conf.setJavaHome(javaHome); } + String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim(); + if(!javaHomeForTests.isEmpty()) { + conf.setJavaHomeForTests(javaHomeForTests); + } String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim(); if(!antEnvOpts.isEmpty()) { conf.setAntEnvOpts(antEnvOpts); Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java (original) +++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java Fri Sep 27 15:00:31 2013 @@ -32,19 +32,21 @@ import com.google.common.base.Strings; import com.google.common.collect.Maps; public class TestConfiguration { + public static final String REPOSITORY = "repository"; + public static final String REPOSITORY_NAME = "repositoryName"; + public static final String BRANCH = "branch"; + public static final String JAVA_HOME = "javaHome"; + public static final String JAVA_HOME_TEST = "javaHomeForTests"; + public static final String ANT_ENV_OPTS = "antEnvOpts"; + private static final String REPOSITORY_TYPE = "repositoryType"; private static final String GIT = "git"; private static final String SVN = "svn"; - private static final String REPOSITORY = "repository"; - private static final String REPOSITORY_NAME = "repositoryName"; - private static final String BRANCH = "branch"; private static final String ANT_ARGS = "antArgs"; - private static final String ANT_ENV_OPTS = "antEnvOpts"; - private static final String JAVA_HOME = "javaHome"; private static final String JIRA_URL = "jiraUrl"; private static final String JIRA_USER = "jiraUser"; private static final String JIRA_PASSWORD = "jiraPassword"; - public static final String JENKINS_URL = "jenkinsURL"; + private static final String JENKINS_URL = "jenkinsURL"; private final Context context; private String antArgs; @@ -54,6 +56,7 @@ public class TestConfiguration { private String repositoryName; private String patch; private String javaHome; + private String javaHomeForTests; private String branch; private final String jenkinsURL; private final String jiraUrl; @@ -79,6 +82,7 @@ public class TestConfiguration { antArgs = Preconditions.checkNotNull(context.getString(ANT_ARGS), ANT_ARGS).trim(); antEnvOpts = context.getString(ANT_ENV_OPTS, "").trim(); javaHome = context.getString(JAVA_HOME, "").trim(); + javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim(); patch = Strings.nullToEmpty(null); jiraName = Strings.nullToEmpty(null); jiraUrl = context.getString(JIRA_URL, "").trim(); @@ -132,6 +136,9 @@ public class TestConfiguration { public String getJavaHome() { return javaHome; } + public String getJavaHomeForTests() { + return javaHomeForTests; + } public String getPatch() { return patch; } @@ -150,6 +157,9 @@ public class TestConfiguration { public void setJavaHome(String javaHome) { this.javaHome = Strings.nullToEmpty(javaHome); } + public void setJavaHomeForTests(String javaHomeForTests) { + this.javaHomeForTests = javaHomeForTests; + } public void setAntArgs(String antArgs) { this.antArgs = Strings.nullToEmpty(antArgs); } @@ -161,10 +171,14 @@ public class TestConfiguration { } @Override public String toString() { - return "Configuration [context=" + context + ", antArgs=" + antArgs - + ", antEnvOpts=" + antEnvOpts + ", repository=" + repository - + ", repositoryName=" + repositoryName + ", patch=" + patch - + ", javaHome=" + javaHome + ", branch=" + branch + "]"; + return "TestConfiguration [antArgs=" + antArgs + ", antEnvOpts=" + + antEnvOpts + ", repositoryType=" + repositoryType + ", repository=" + + repository + ", repositoryName=" + repositoryName + ", patch=" + + patch + ", javaHome=" + javaHome + ", javaHomeForTests=" + + javaHomeForTests + ", branch=" + branch + ", jenkinsURL=" + + jenkinsURL + ", jiraUrl=" + jiraUrl + ", jiraUser=" + jiraUser + + ", jiraName=" + jiraName + ", clearLibraryCache=" + clearLibraryCache + + "]"; } public static TestConfiguration fromInputStream(InputStream inputStream, Logger logger) throws IOException { Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java (original) +++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java Fri Sep 27 15:00:31 2013 @@ -106,7 +106,7 @@ public class CloudExecutionContextProvid return size() > 100; } }); - mTerminationExecutor = Executors.newCachedThreadPool(); + mTerminationExecutor = Executors.newSingleThreadExecutor(); mHostLog = new RandomAccessFile(new File(dataDir, "hosts"), "rw"); initialize(); } Modified: hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm (original) +++ hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm Fri Sep 27 15:00:31 2013 @@ -23,7 +23,11 @@ chmod -R u+w $logDir rm -rf $logDir # makes $logDir and $logDir/tmp mkdir -p $logDir/tmp -if [[ -n "${javaHome}" ]] +if [[ -n "${javaHomeForTests}" ]] +then + export JAVA_HOME=$javaHomeForTests + export PATH=$JAVA_HOME/bin/:$PATH +elif [[ -n "${javaHome}" ]] then export JAVA_HOME=$javaHome export PATH=$JAVA_HOME/bin/:$PATH Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java (original) +++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java Fri Sep 27 15:00:31 2013 @@ -78,6 +78,30 @@ public class TestScripts { Approvals.verify(actual); } @Test + public void testAlternativeTestJVM() throws Throwable { + Map templateVariables = Maps.newHashMap(); + templateVariables.put("repository", "git:///repo1"); + templateVariables.put("repositoryName", "apache"); + templateVariables.put("branch", "branch-1"); + templateVariables.put("localDir", "/some/local/dir"); + templateVariables.put("workingDir", "/some/working/dir"); + templateVariables.put("antArgs", "-Dant=arg1"); + templateVariables.put("buildTag", "build-1"); + templateVariables.put("logDir", "/some/log/dir"); + templateVariables.put("instanceName", "instance-1"); + templateVariables.put("batchName","batch-1"); + templateVariables.put("numOfFailedTests", "20"); + templateVariables.put("maxSourceDirs", String.valueOf(5)); + templateVariables.put("testArguments", "-Dtest=arg1"); + templateVariables.put("clearLibraryCache", "true"); + templateVariables.put("javaHome", "/usr/java/jdk1.7"); + templateVariables.put("javaHomeForTests", "/usr/java/jdk1.7-other"); + templateVariables.put("antEnvOpts", "-Dhttp.proxyHost=somehost -Dhttp.proxyPort=3128"); + String template = readResource("batch-exec.vm"); + String actual = getTemplateResult(template, templateVariables); + Approvals.verify(actual); + } + @Test public void testPrepNone() throws Throwable { Map templateVariables = Maps.newHashMap(); templateVariables.put("repository", "git:///repo1"); Added: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt?rev=1526925&view=auto ============================================================================== --- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt (added) +++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt Fri Sep 27 15:00:31 2013 @@ -0,0 +1,69 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +umask 0022 +find /some/local/dir/instance-1/apache-source/build -name 'TEST-*.xml' -delete +find /some/local/dir/instance-1/apache-source/build -name 'hive.log' -delete +chmod -R u+w /some/log/dir +rm -rf /some/log/dir +# makes /some/log/dir and /some/log/dir/tmp +mkdir -p /some/log/dir/tmp +if [[ -n "/usr/java/jdk1.7-other" ]] +then + export JAVA_HOME=/usr/java/jdk1.7-other + export PATH=$JAVA_HOME/bin/:$PATH +elif [[ -n "/usr/java/jdk1.7" ]] +then + export JAVA_HOME=/usr/java/jdk1.7 + export PATH=$JAVA_HOME/bin/:$PATH +fi +export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=/some/log/dir/tmp -Dhttp.proxyHost=somehost -Dhttp.proxyPort=3128" +export HADOOP_ROOT_LOGGER=INFO,console +export HADOOP_OPTS="-Dhive.log.dir=/some/log/dir -Dhive.query.id=hadoop -Djava.io.tmpdir=/some/log/dir/tmp" +cd /some/local/dir/instance-1/apache-source && \ + timeout 2h ant test -Dtest.junit.output.format=xml -Dant=arg1 \ + -Divy.default.ivy.user.dir=/some/local/dir/instance-1/ivy \ + -Divy.default.always.check.exact.revision=false \ + -Divy.cache.ttl.default=eternal -Divy.checkmodified=false \ + -Dmvn.local.repo=/some/local/dir/instance-1/maven \ + -Dant=arg1 -Dtest=arg1 1>/some/log/dir/ant-test.txt 2>&1 +ret=$? +if [[ $ret -ne 0 ]] +then + if [[ 20 -lt 5 ]] + then + cp -R /some/local/dir/instance-1/apache-source /some/log/dir/source + else + echo "Number of failed tests 20 exceeded threshold, not copying source" + fi +fi +if [[ -f /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log ]] +then + mv /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log /some/log/dir/hive.log +fi +batchName=batch-1 +for testOutputFile in $(find /some/local/dir/instance-1/apache-source/ -name 'TEST-*.xml') +do + mv "$testOutputFile" "/some/log/dir/TEST-$batchName-$(basename $testOutputFile)" +done +if [[ -f /some/log/dir/.log ]] +then + mv /some/log/dir/.log /some/log/dir/dot.log +else + echo "/some/log/dir/.log does not exist" +fi +exit $ret Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1526925&r1=1526924&r2=1526925&view=diff ============================================================================== --- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt (original) +++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt Fri Sep 27 15:00:31 2013 @@ -22,7 +22,11 @@ chmod -R u+w /some/log/dir rm -rf /some/log/dir # makes /some/log/dir and /some/log/dir/tmp mkdir -p /some/log/dir/tmp -if [[ -n "/usr/java/jdk1.7" ]] +if [[ -n "${javaHomeForTests}" ]] +then + export JAVA_HOME=$javaHomeForTests + export PATH=$JAVA_HOME/bin/:$PATH +elif [[ -n "/usr/java/jdk1.7" ]] then export JAVA_HOME=/usr/java/jdk1.7 export PATH=$JAVA_HOME/bin/:$PATH