hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject [01/44] hive git commit: HIVE-9726 - Upgrade to spark 1.3 [Spark Branch]
Date Thu, 23 Apr 2015 02:34:10 GMT
Repository: hive
Updated Branches:
  refs/heads/master 7eaccc3d3 -> 864d7e98d


HIVE-9726 - Upgrade to spark 1.3 [Spark Branch]

git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1661265 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0b638756
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0b638756
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0b638756

Branch: refs/heads/master
Commit: 0b63875648c8a847ca3b99cb60e5248c1646e155
Parents: 36c9425
Author: Brock Noland <brock@apache.org>
Authored: Sat Feb 21 01:21:31 2015 +0000
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Wed Apr 22 19:33:48 2015 -0700

----------------------------------------------------------------------
 pom.xml                                                 | 12 +++++++++++-
 .../ql/exec/spark/status/impl/JobMetricsListener.java   | 12 ++++++++++++
 .../org/apache/hadoop/hive/shims/Hadoop23Shims.java     |  1 +
 .../java/org/apache/hive/spark/client/RemoteDriver.java | 12 ++++++++++++
 4 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0b638756/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0839285..0d4f2dd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -156,7 +156,7 @@
     <ST4.version>4.0.4</ST4.version>
     <tez.version>0.5.2</tez.version>
     <super-csv.version>2.2.0</super-csv.version>
-    <spark.version>1.2.0</spark.version>
+    <spark.version>1.3.0</spark.version>
     <scala.binary.version>2.10</scala.binary.version>
     <scala.version>2.10.4</scala.version>
     <tempus-fugit.version>1.1</tempus-fugit.version>
@@ -216,6 +216,16 @@
          <enabled>false</enabled>
        </snapshots>
     </repository>
+     <repository>
+       <id>spark-1.3</id>
+       <url>http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark_2.10-1.3-rc1/</url>
+       <releases>
+         <enabled>true</enabled>
+       </releases>
+       <snapshots>
+         <enabled>false</enabled>
+       </snapshots>
+    </repository>
   </repositories>
 
   <!-- Hadoop dependency management is done at the bottom under profiles -->

http://git-wip-us.apache.org/repos/asf/hive/blob/0b638756/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java
index 8243d4d..51772cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java
@@ -39,6 +39,8 @@ import org.apache.spark.scheduler.SparkListenerTaskEnd;
 import org.apache.spark.scheduler.SparkListenerTaskGettingResult;
 import org.apache.spark.scheduler.SparkListenerTaskStart;
 import org.apache.spark.scheduler.SparkListenerUnpersistRDD;
+import org.apache.spark.scheduler.SparkListenerExecutorRemoved;
+import org.apache.spark.scheduler.SparkListenerExecutorAdded;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -52,6 +54,16 @@ public class JobMetricsListener implements SparkListener {
   private final Map<Integer, Map<String, List<TaskMetrics>>> allJobMetrics
= Maps.newHashMap();
 
   @Override
+  public void onExecutorRemoved(SparkListenerExecutorRemoved removed) {
+
+  }
+
+  @Override
+  public void onExecutorAdded(SparkListenerExecutorAdded added) {
+
+  }
+
+  @Override
   public void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
 
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/0b638756/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index b38f59f..997e49d 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -450,6 +450,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
 
       mr = new MiniSparkOnYARNCluster("sparkOnYarn");
       conf.set("fs.defaultFS", nameNode);
+      conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
       mr.init(conf);
       mr.start();
       this.conf = mr.getConfig();

http://git-wip-us.apache.org/repos/asf/hive/blob/0b638756/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
index 8bda02b..c2ac0c2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
@@ -57,6 +57,8 @@ import org.apache.spark.scheduler.SparkListenerTaskEnd;
 import org.apache.spark.scheduler.SparkListenerTaskGettingResult;
 import org.apache.spark.scheduler.SparkListenerTaskStart;
 import org.apache.spark.scheduler.SparkListenerUnpersistRDD;
+import org.apache.spark.scheduler.SparkListenerExecutorRemoved;
+import org.apache.spark.scheduler.SparkListenerExecutorAdded;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -425,6 +427,16 @@ public class RemoteDriver {
     private final Map<Integer, Integer> stageToJobId = Maps.newHashMap();
 
     @Override
+    public void onExecutorRemoved(SparkListenerExecutorRemoved removed) {
+
+    }
+
+    @Override
+    public void onExecutorAdded(SparkListenerExecutorAdded added) {
+
+    }
+
+    @Override
     public void onJobStart(SparkListenerJobStart jobStart) {
       synchronized (stageToJobId) {
         for (int i = 0; i < jobStart.stageIds().length(); i++) {


Mime
View raw message