hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1526925 - in /hive/trunk/testutils/ptest2/src: main/java/org/apache/hive/ptest/execution/ main/java/org/apache/hive/ptest/execution/conf/ main/java/org/apache/hive/ptest/execution/context/ main/resources/ test/java/org/apache/hive/ptest/ex...
Date Fri, 27 Sep 2013 15:00:31 GMT
Author: brock
Date: Fri Sep 27 15:00:31 2013
New Revision: 1526925

URL: http://svn.apache.org/r1526925
Log:
HIVE-5361 - PTest2 should allow a different JVM for compilation versus execution

Added:
    hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
Removed:
    hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java
    hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestCleanupPhase.java
Modified:
    hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
    hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
    hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
    hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm
    hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
    hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt

Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Fri
Sep 27 15:00:31 2013
@@ -110,6 +110,7 @@ public class PTest {
     put("buildTag", buildTag).
     put("logDir", logDir.getAbsolutePath()).
     put("javaHome", configuration.getJavaHome()).
+    put("javaHomeForTests", configuration.getJavaHomeForTests()).
     put("antEnvOpts", configuration.getAntEnvOpts());
     final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
     TestParser testParser = new TestParser(configuration.getContext(),
@@ -221,12 +222,13 @@ public class PTest {
   }
 
   private static final String PROPERTIES = "properties";
-  private static final String REPOSITORY = "repository";
-  private static final String REPOSITORY_NAME = "repositoryName";
-  private static final String BRANCH = "branch";
+  private static final String REPOSITORY = TestConfiguration.REPOSITORY;
+  private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME;
+  private static final String BRANCH = TestConfiguration.BRANCH;
   private static final String PATCH = "patch";
-  private static final String JAVA_HOME = "javaHome";
-  private static final String ANT_ENV_OPTS = "antEnvOpts";
+  private static final String JAVA_HOME = TestConfiguration.JAVA_HOME;
+  private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST;
+  private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS;
   /**
    * All args override properties file settings except
    * for this one which is additive.
@@ -243,7 +245,8 @@ public class PTest {
     options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
     options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
     options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments");
-    options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests");
+    options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless
" + JAVA_HOME_TEST + " is specified)");
+    options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
     options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environemnt variable setting");
     CommandLine commandLine = parser.parse(options, args);
     if(!commandLine.hasOption(PROPERTIES)) {
@@ -282,6 +285,10 @@ public class PTest {
         if(!javaHome.isEmpty()) {
           conf.setJavaHome(javaHome);
         }
+        String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
+        if(!javaHomeForTests.isEmpty()) {
+          conf.setJavaHomeForTests(javaHomeForTests);
+        }
         String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
         if(!antEnvOpts.isEmpty()) {
           conf.setAntEnvOpts(antEnvOpts);

Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
(original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
Fri Sep 27 15:00:31 2013
@@ -32,19 +32,21 @@ import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
 
 public class TestConfiguration {
+  public static final String REPOSITORY = "repository";
+  public static final String REPOSITORY_NAME = "repositoryName";
+  public static final String BRANCH = "branch";
+  public static final String JAVA_HOME = "javaHome";
+  public static final String JAVA_HOME_TEST = "javaHomeForTests";
+  public static final String ANT_ENV_OPTS = "antEnvOpts";
+  
   private static final String REPOSITORY_TYPE = "repositoryType";
   private static final String GIT = "git";
   private static final String SVN = "svn";
-  private static final String REPOSITORY = "repository";
-  private static final String REPOSITORY_NAME = "repositoryName";
-  private static final String BRANCH = "branch";
   private static final String ANT_ARGS = "antArgs";
-  private static final String ANT_ENV_OPTS = "antEnvOpts";
-  private static final String JAVA_HOME = "javaHome";
   private static final String JIRA_URL = "jiraUrl";
   private static final String JIRA_USER = "jiraUser";
   private static final String JIRA_PASSWORD = "jiraPassword";
-  public static final String JENKINS_URL = "jenkinsURL";
+  private static final String JENKINS_URL = "jenkinsURL";
 
   private final Context context;
   private String antArgs;
@@ -54,6 +56,7 @@ public class TestConfiguration {
   private String repositoryName;
   private String patch;
   private String javaHome;
+  private String javaHomeForTests;
   private String branch;
   private final String jenkinsURL;
   private final String jiraUrl;
@@ -79,6 +82,7 @@ public class TestConfiguration {
     antArgs =  Preconditions.checkNotNull(context.getString(ANT_ARGS), ANT_ARGS).trim();
     antEnvOpts =  context.getString(ANT_ENV_OPTS, "").trim();
     javaHome =  context.getString(JAVA_HOME, "").trim();
+    javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim();
     patch = Strings.nullToEmpty(null);
     jiraName = Strings.nullToEmpty(null);
     jiraUrl = context.getString(JIRA_URL, "").trim();
@@ -132,6 +136,9 @@ public class TestConfiguration {
   public String getJavaHome() {
     return javaHome;
   }
+  public String getJavaHomeForTests() {
+    return javaHomeForTests;
+  }
   public String getPatch() {
     return patch;
   }
@@ -150,6 +157,9 @@ public class TestConfiguration {
   public void setJavaHome(String javaHome) {
     this.javaHome = Strings.nullToEmpty(javaHome);
   }
+  public void setJavaHomeForTests(String javaHomeForTests) {
+      this.javaHomeForTests = javaHomeForTests;
+  }
   public void setAntArgs(String antArgs) {
     this.antArgs = Strings.nullToEmpty(antArgs);
   }
@@ -161,10 +171,14 @@ public class TestConfiguration {
   }
   @Override
   public String toString() {
-    return "Configuration [context=" + context + ", antArgs=" + antArgs
-        + ", antEnvOpts=" + antEnvOpts + ", repository=" + repository
-        + ", repositoryName=" + repositoryName + ", patch=" + patch
-        + ", javaHome=" + javaHome + ", branch=" + branch + "]";
+    return "TestConfiguration [antArgs=" + antArgs + ", antEnvOpts="
+        + antEnvOpts + ", repositoryType=" + repositoryType + ", repository="
+        + repository + ", repositoryName=" + repositoryName + ", patch="
+        + patch + ", javaHome=" + javaHome + ", javaHomeForTests="
+        + javaHomeForTests + ", branch=" + branch + ", jenkinsURL="
+        + jenkinsURL + ", jiraUrl=" + jiraUrl + ", jiraUser=" + jiraUser
+        + ", jiraName=" + jiraName + ", clearLibraryCache=" + clearLibraryCache
+        + "]";
   }
   public static TestConfiguration fromInputStream(InputStream inputStream, Logger logger)
       throws IOException {

Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
(original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
Fri Sep 27 15:00:31 2013
@@ -106,7 +106,7 @@ public class CloudExecutionContextProvid
             return size() > 100;
           }
         });
-    mTerminationExecutor = Executors.newCachedThreadPool();
+    mTerminationExecutor = Executors.newSingleThreadExecutor();
     mHostLog = new RandomAccessFile(new File(dataDir, "hosts"), "rw");
     initialize();
   }

Modified: hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm (original)
+++ hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm Fri Sep 27 15:00:31 2013
@@ -23,7 +23,11 @@ chmod -R u+w $logDir
 rm -rf $logDir
 # makes $logDir and $logDir/tmp
 mkdir -p $logDir/tmp
-if [[ -n "${javaHome}" ]]
+if [[ -n "${javaHomeForTests}" ]]
+then
+  export JAVA_HOME=$javaHomeForTests
+  export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "${javaHome}" ]]
 then
   export JAVA_HOME=$javaHome
   export PATH=$JAVA_HOME/bin/:$PATH

Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
(original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
Fri Sep 27 15:00:31 2013
@@ -78,6 +78,30 @@ public class TestScripts  {
     Approvals.verify(actual);
   }
   @Test
+  public void testAlternativeTestJVM() throws Throwable {
+    Map<String, String> templateVariables = Maps.newHashMap();
+    templateVariables.put("repository", "git:///repo1");
+    templateVariables.put("repositoryName", "apache");
+    templateVariables.put("branch", "branch-1");
+    templateVariables.put("localDir", "/some/local/dir");
+    templateVariables.put("workingDir", "/some/working/dir");
+    templateVariables.put("antArgs", "-Dant=arg1");
+    templateVariables.put("buildTag", "build-1");
+    templateVariables.put("logDir", "/some/log/dir");
+    templateVariables.put("instanceName", "instance-1");
+    templateVariables.put("batchName","batch-1");
+    templateVariables.put("numOfFailedTests", "20");
+    templateVariables.put("maxSourceDirs", String.valueOf(5));
+    templateVariables.put("testArguments", "-Dtest=arg1");
+    templateVariables.put("clearLibraryCache", "true");
+    templateVariables.put("javaHome", "/usr/java/jdk1.7");
+    templateVariables.put("javaHomeForTests", "/usr/java/jdk1.7-other");
+    templateVariables.put("antEnvOpts", "-Dhttp.proxyHost=somehost -Dhttp.proxyPort=3128");
+    String template = readResource("batch-exec.vm");
+    String actual = getTemplateResult(template, templateVariables);
+    Approvals.verify(actual);
+  }
+  @Test
   public void testPrepNone() throws Throwable {
     Map<String, String> templateVariables = Maps.newHashMap();
     templateVariables.put("repository", "git:///repo1");

Added: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt?rev=1526925&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
(added)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
Fri Sep 27 15:00:31 2013
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+umask 0022
+find /some/local/dir/instance-1/apache-source/build -name 'TEST-*.xml' -delete
+find /some/local/dir/instance-1/apache-source/build -name 'hive.log' -delete
+chmod -R u+w /some/log/dir
+rm -rf /some/log/dir
+# makes /some/log/dir and /some/log/dir/tmp
+mkdir -p /some/log/dir/tmp
+if [[ -n "/usr/java/jdk1.7-other" ]]
+then
+  export JAVA_HOME=/usr/java/jdk1.7-other
+  export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "/usr/java/jdk1.7" ]]
+then
+  export JAVA_HOME=/usr/java/jdk1.7
+  export PATH=$JAVA_HOME/bin/:$PATH
+fi
+export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=/some/log/dir/tmp -Dhttp.proxyHost=somehost
-Dhttp.proxyPort=3128"
+export HADOOP_ROOT_LOGGER=INFO,console
+export HADOOP_OPTS="-Dhive.log.dir=/some/log/dir -Dhive.query.id=hadoop -Djava.io.tmpdir=/some/log/dir/tmp"
+cd /some/local/dir/instance-1/apache-source && \
+  timeout 2h ant test -Dtest.junit.output.format=xml -Dant=arg1 \
+   -Divy.default.ivy.user.dir=/some/local/dir/instance-1/ivy \
+   -Divy.default.always.check.exact.revision=false \
+   -Divy.cache.ttl.default=eternal -Divy.checkmodified=false \
+   -Dmvn.local.repo=/some/local/dir/instance-1/maven \
+   -Dant=arg1 -Dtest=arg1 1>/some/log/dir/ant-test.txt 2>&1
+ret=$?
+if [[ $ret -ne 0 ]]
+then
+  if [[ 20 -lt 5 ]]
+  then
+    cp -R /some/local/dir/instance-1/apache-source /some/log/dir/source
+  else
+    echo "Number of failed tests 20 exceeded threshold, not copying source"
+  fi
+fi
+if [[ -f /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log ]]
+then
+  mv /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log /some/log/dir/hive.log
+fi
+batchName=batch-1
+for testOutputFile in $(find /some/local/dir/instance-1/apache-source/ -name 'TEST-*.xml')
+do
+  mv "$testOutputFile" "/some/log/dir/TEST-$batchName-$(basename $testOutputFile)"
+done
+if [[ -f /some/log/dir/.log ]]
+then
+  mv /some/log/dir/.log /some/log/dir/dot.log
+else
+  echo "/some/log/dir/.log does not exist"
+fi
+exit $ret

Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
(original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
Fri Sep 27 15:00:31 2013
@@ -22,7 +22,11 @@ chmod -R u+w /some/log/dir
 rm -rf /some/log/dir
 # makes /some/log/dir and /some/log/dir/tmp
 mkdir -p /some/log/dir/tmp
-if [[ -n "/usr/java/jdk1.7" ]]
+if [[ -n "${javaHomeForTests}" ]]
+then
+  export JAVA_HOME=$javaHomeForTests
+  export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "/usr/java/jdk1.7" ]]
 then
   export JAVA_HOME=/usr/java/jdk1.7
   export PATH=$JAVA_HOME/bin/:$PATH



Mime
View raw message