hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1538880 [46/46] - in /hive/branches/tez: ./ ant/ ant/src/org/apache/hadoop/hive/ant/ beeline/ beeline/src/java/org/apache/hive/beeline/ beeline/src/main/ beeline/src/test/org/apache/hive/beeline/src/test/ cli/ common/ common/src/java/conf/...
Date Tue, 05 Nov 2013 07:01:58 GMT
Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
(original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
Tue Nov  5 07:01:32 2013
@@ -77,6 +77,8 @@ public class PTest {
   private final Logger mLogger;
   private final List<HostExecutor> mHostExecutors;
   private final String mBuildTag;
+  private final SSHCommandExecutor mSshCommandExecutor;
+  private final RSyncCommandExecutor mRsyncCommandExecutor;
 
   public PTest(final TestConfiguration configuration, final ExecutionContext executionContext,
       final String buildTag, final File logDir, final LocalCommandFactory localCommandFactory,
@@ -88,6 +90,8 @@ public class PTest {
     mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
     mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
     mExecutionContext = executionContext;
+    mSshCommandExecutor = sshCommandExecutor;
+    mRsyncCommandExecutor = rsyncCommandExecutor;
     mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
     final File failedLogDir = Dirs.create(new File(logDir, "failed"));
     final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
@@ -171,10 +175,13 @@ public class PTest {
       error = true;
     } finally {
       for(HostExecutor hostExecutor : mHostExecutors) {
+        hostExecutor.shutdownNow();
         if(hostExecutor.isBad()) {
           mExecutionContext.addBadHost(hostExecutor.getHost());
         }
       }
+      mSshCommandExecutor.shutdownNow();
+      mRsyncCommandExecutor.shutdownNow();
       mExecutor.shutdownNow();
       SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
       if(failedTests.isEmpty()) {

Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java
(original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java
Tue Nov  5 07:01:32 2013
@@ -22,7 +22,9 @@ import java.io.IOException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hive.ptest.execution.Constants;
 import org.apache.hive.ptest.execution.LocalCommand;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
 import org.apache.hive.ptest.execution.LocalCommand.CollectPolicy;
 import org.slf4j.Logger;
 
@@ -30,12 +32,20 @@ import org.slf4j.Logger;
 public class RSyncCommandExecutor {
 
   private final Logger mLogger;
+  private final LocalCommandFactory mLocalCommandFactory;
   private final Semaphore mSemaphore;
+  private volatile boolean mShutdown;
 
-  public RSyncCommandExecutor(Logger logger) {
+  public RSyncCommandExecutor(Logger logger, LocalCommandFactory localCommandFactory) {
     mLogger = logger;
-    mSemaphore = new Semaphore(5);
+    mLocalCommandFactory = localCommandFactory;
+    mSemaphore = new Semaphore(Math.min(Runtime.getRuntime().availableProcessors() * 5, 10));
+    mShutdown = false;
+  }
+  public RSyncCommandExecutor(Logger logger) {
+    this(logger, new LocalCommandFactory(logger));
   }
+
   /**
    * Execute the given RSync. If the command exits with a non-zero
    * exit status the command will be retried up to three times.
@@ -52,25 +62,31 @@ public class RSyncCommandExecutor {
       do {
         retry = false;
         if(command.getType() == RSyncCommand.Type.TO_LOCAL) {
-          cmd = new LocalCommand(mLogger, collector,
+          cmd = mLocalCommandFactory.create(collector,
               String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 %s@%s:%s
%s",
                   command.getPrivateKey(), command.getUser(), command.getHost(),
                   command.getRemoteFile(), command.getLocalFile()));
         } else if(command.getType() == RSyncCommand.Type.FROM_LOCAL) {
-          cmd = new LocalCommand(mLogger, collector,
+          cmd = mLocalCommandFactory.create(collector,
               String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 --delete
--delete-during --force %s %s@%s:%s",
                   command.getPrivateKey(), command.getLocalFile(), command.getUser(), command.getHost(),
                   command.getRemoteFile()));
         } else {
           throw new UnsupportedOperationException(String.valueOf(command.getType()));
         }
+        if(mShutdown) {
+          mLogger.warn("Shutting down command " + command);
+          cmd.kill();
+          command.setExitCode(Constants.EXIT_CODE_UNKNOWN);
+          return;
+        }
         // 12 is timeout and 255 is unspecified error
         if(attempts++ <= 3 && cmd.getExitCode() != 0) {
           mLogger.warn("Command exited with " + cmd.getExitCode() + ", will retry: " + command);
           retry = true;
           TimeUnit.SECONDS.sleep(20);
         }
-      } while (retry); // an error occurred, re-try
+      } while (!mShutdown && retry); // an error occurred, re-try
       command.setExitCode(cmd.getExitCode());
     } catch (IOException e) {
       command.setException(e);
@@ -83,4 +99,10 @@ public class RSyncCommandExecutor {
       command.setOutput(collector.getOutput());
     }
   }
-}
+  boolean isShutdown() {
+    return mShutdown;
+  }
+  public void shutdownNow() {
+    this.mShutdown = true;
+  }
+}
\ No newline at end of file

Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
(original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
Tue Nov  5 07:01:32 2013
@@ -23,14 +23,22 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hive.ptest.execution.Constants;
 import org.apache.hive.ptest.execution.LocalCommand;
 import org.apache.hive.ptest.execution.LocalCommand.CollectPolicy;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
 import org.slf4j.Logger;
 
 public class SSHCommandExecutor {
 
   private final Logger mLogger;
-
-  public SSHCommandExecutor(Logger logger) {
+  private final LocalCommandFactory mLocalCommandFactory;
+  private volatile boolean mShutdown;
+  
+  public SSHCommandExecutor(Logger logger, LocalCommandFactory localCommandFactory) {
     mLogger = logger;
+    mShutdown = false;
+    mLocalCommandFactory = localCommandFactory;
+  }
+  public SSHCommandExecutor(Logger logger) {
+    this(logger, new LocalCommandFactory(logger));
   }
   /**
    * Execute the given command via the ssh command line tool. If the command
@@ -46,13 +54,19 @@ public class SSHCommandExecutor {
       LocalCommand cmd;
       do {
         retry = false;
-        cmd = new LocalCommand(mLogger, collector, commandText);
+        cmd = mLocalCommandFactory.create(collector, commandText);
+        if(mShutdown) {
+          mLogger.warn("Shutting down command " + command);
+          cmd.kill();
+          command.setExitCode(Constants.EXIT_CODE_UNKNOWN);
+          return;
+        }
         if(attempts++ <= 3 && cmd.getExitCode() == Constants.EXIT_CODE_UNKNOWN)
{
           mLogger.warn("Command exited with " + cmd.getExitCode() + ", will retry: " + command);
           retry = true;
           TimeUnit.SECONDS.sleep(5);
         }
-      } while (retry); // an error occurred, re-try
+      } while (!mShutdown && retry); // an error occurred, re-try
       command.setExitCode(cmd.getExitCode());
     } catch (Exception e) {
       if(command.getExitCode() == Constants.EXIT_CODE_SUCCESS) {
@@ -63,4 +77,10 @@ public class SSHCommandExecutor {
       command.setOutput(collector.getOutput());
     }
   }
-}
+  boolean isShutdown() {
+    return mShutdown;
+  }
+  public void shutdownNow() {
+    this.mShutdown = true;
+  }
+}
\ No newline at end of file

Modified: hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties (original)
+++ hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties Tue Nov  5 07:01:32
2013
@@ -31,3 +31,7 @@ log4j.logger.org.jclouds=INFO
 log4j.logger.jclouds=INFO
 log4j.logger.org.apache.hive=DEBUG
 log4j.logger.org.apache.http=TRACE
+
+# Silence useless ZK logs
+log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
Tue Nov  5 07:01:32 2013
@@ -193,6 +193,20 @@ public class TestHostExecutor {
     Approvals.verify(getExecutedCommands());
   }
   @Test
+  public void testShutdownBeforeExec()
+      throws Exception {
+    rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh
"
+        + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh", Constants.EXIT_CODE_UNKNOWN);
+    HostExecutor executor = createHostExecutor();
+    parallelWorkQueue.addAll(Lists.newArrayList(testBatchParallel1));
+    executor.shutdownNow();
+    executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
+    Assert.assertEquals(Collections.emptySet(),  failedTestResults);
+    Assert.assertEquals(parallelWorkQueue.toString(), 1, parallelWorkQueue.size());
+    Approvals.verify(getExecutedCommands());
+    Assert.assertTrue(executor.isShutdown());
+  }
+  @Test
   public void testIsolatedFailsOnRsyncUnknown()
       throws Exception {
     rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh
"+
@@ -201,7 +215,7 @@ public class TestHostExecutor {
     isolatedWorkQueue.addAll(Lists.newArrayList(testBatchIsolated1));
     executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
     Assert.assertEquals(Collections.emptySet(),  failedTestResults);
-    Assert.assertTrue(isolatedWorkQueue.toString(), parallelWorkQueue.isEmpty());
+    Assert.assertTrue(isolatedWorkQueue.toString(), isolatedWorkQueue.isEmpty());
     Approvals.verify(getExecutedCommands());
   }
   @Test

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java
Tue Nov  5 07:01:32 2013
@@ -18,9 +18,10 @@
  */
 package org.apache.hive.ptest.execution;
 
+import java.util.List;
 import junit.framework.Assert;
-
 import org.junit.Test;
+import com.google.common.collect.Lists;
 
 public class TestJIRAService  {
 
@@ -41,4 +42,30 @@ public class TestJIRAService  {
   public void testFormatBuildTagNoDashNone() throws Throwable {
     JIRAService.formatBuildTag("abc123");
   }
+  @Test
+  public void testTrimMesssagesBoundry() {
+    List<String> messages = Lists.newArrayList();
+    Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+    messages.clear();
+    for (int i = 0; i < JIRAService.MAX_MESSAGES; i++) {
+      messages.add(String.valueOf(i));
+    }
+    Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+  }
+  @Test
+  public void testTrimMesssagesNotTrimmed() {
+    List<String> messages = Lists.newArrayList("a", "b", "c");
+    Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+  }
+  @Test
+  public void testTrimMesssagesTrimmed() {
+    List<String> messages = Lists.newArrayList();
+    for (int i = 0; i < JIRAService.MAX_MESSAGES + 1; i++) {
+      messages.add(String.valueOf(i));
+    }
+    List<String> expected = Lists.newArrayList(messages);
+    expected.remove(0);
+    expected.add(0, JIRAService.TRIMMED_MESSAGE);
+    Assert.assertEquals(expected, JIRAService.trimMessages(messages));
+  }
 }
\ No newline at end of file

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
Tue Nov  5 07:01:32 2013
@@ -61,7 +61,7 @@ then
   	testModule=./
   fi
   pushd $testModule
-  timeout 2h mvn -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
+  timeout 2h mvn -B -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
     $mavenArgs $mavenTestArgs -Dtest=arg1 1>/some/log/dir/maven-test.txt 2>&1 </dev/null
&
 
   pid=$!

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
Tue Nov  5 07:01:32 2013
@@ -61,7 +61,7 @@ then
   	testModule=./
   fi
   pushd $testModule
-  timeout 2h mvn -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
+  timeout 2h mvn -B -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
     $mavenArgs $mavenTestArgs -Dtest=arg1 1>/some/log/dir/maven-test.txt 2>&1 </dev/null
&
 
   pid=$!

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
Tue Nov  5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
   if [[ "${buildTool}" == "maven" ]]
   then
     rm -rf /some/working/dir/maven/org/apache/hive
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
     cd itests
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
   elif [[ "${buildTool}" == "ant" ]]
   then
     ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
Tue Nov  5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
   if [[ "ant" == "maven" ]]
   then
     rm -rf /some/working/dir/maven/org/apache/hive
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
     cd itests
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
   elif [[ "ant" == "ant" ]]
   then
     ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \

Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
(original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
Tue Nov  5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
   if [[ "maven" == "maven" ]]
   then
     rm -rf /some/working/dir/maven/org/apache/hive
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
     cd itests
-    mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
-    mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+    mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+    mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
   elif [[ "maven" == "ant" ]]
   then
     ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \



Mime
View raw message