hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1643157 - in /hive/branches/spark: data/conf/spark/hive-site.xml spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Date Fri, 05 Dec 2014 06:06:49 GMT
Author: xuefu
Date: Fri Dec  5 06:06:49 2014
New Revision: 1643157

URL: http://svn.apache.org/viewvc?rev=1643157&view=rev
Log:
HIVE-8991: Fix custom_input_output_format [Spark Branch] (Rui via Xuefu)

Modified:
    hive/branches/spark/data/conf/spark/hive-site.xml
    hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
    hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java

Modified: hive/branches/spark/data/conf/spark/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/hive-site.xml?rev=1643157&r1=1643156&r2=1643157&view=diff
==============================================================================
--- hive/branches/spark/data/conf/spark/hive-site.xml (original)
+++ hive/branches/spark/data/conf/spark/hive-site.xml Fri Dec  5 06:06:49 2014
@@ -210,4 +210,14 @@
   <value>${spark.home}/logs/</value>
 </property>
 
+<property>
+  <name>spark.driver.extraClassPath</name>
+  <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+</property>
+
+<property>
+  <name>hive.aux.jars.path</name>
+  <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
+</property>
+
 </configuration>

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java?rev=1643157&r1=1643156&r2=1643157&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
(original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
Fri Dec  5 06:06:49 2014
@@ -234,6 +234,7 @@ public class RemoteDriver {
         // Catch throwables in a best-effort to report job status back to the client. It's
         // re-thrown so that the executor can destroy the affected thread (or the JVM can
         // die or whatever would happen if the throwable bubbled up).
+        LOG.info("Failed to run job " + req.id, t);
         client.tell(new Protocol.JobResult(req.id, null, t, null), actor);
         throw new ExecutionException(t);
       } finally {

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1643157&r1=1643156&r2=1643157&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
(original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Fri Dec  5 06:06:49 2014
@@ -55,6 +55,7 @@ class SparkClientImpl implements SparkCl
   private static final Logger LOG = LoggerFactory.getLogger(SparkClientImpl.class);
   
   private static final String DEFAULT_CONNECTION_TIMEOUT = "60"; // In seconds
+  private static final long DEFAULT_SHUTDOWN_TIMEOUT = 10000; // In milliseconds
 
   private final Map<String, String> conf;
   private final AtomicInteger childIdGenerator;
@@ -110,11 +111,16 @@ class SparkClientImpl implements SparkCl
       remoteRef.tell(new Protocol.EndSession(), clientRef);
     }
     unbind(clientRef);
+    long endTime = System.currentTimeMillis() + DEFAULT_SHUTDOWN_TIMEOUT;
     try {
-      driverThread.join(); // TODO: timeout?
+      driverThread.join(DEFAULT_SHUTDOWN_TIMEOUT); // TODO: timeout?
     } catch (InterruptedException ie) {
       LOG.debug("Interrupted before driver thread was finished.");
     }
+    if (endTime - System.currentTimeMillis() <= 0) {
+      LOG.debug("Shut down time out.");
+      driverThread.interrupt();
+    }
   }
 
   @Override
@@ -280,6 +286,8 @@ class SparkClientImpl implements SparkCl
             if (exitCode != 0) {
               LOG.warn("Child process exited with code {}.", exitCode);
             }
+          } catch (InterruptedException ie) {
+            child.destroy();
           } catch (Exception e) {
             LOG.warn("Exception while waiting for child process.", e);
           }



Mime
View raw message