hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject hive git commit: HIVE-12828: Update Spark version to 1.6 (Rui reviewed by Xuefu)
Date Fri, 15 Jan 2016 13:25:42 GMT
Repository: hive
Updated Branches:
  refs/heads/spark d5603ae26 -> 814bdaa52


HIVE-12828: Update Spark version to 1.6 (Rui reviewed by Xuefu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/814bdaa5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/814bdaa5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/814bdaa5

Branch: refs/heads/spark
Commit: 814bdaa52c5f5c6d1ba317cd260cbd463152f451
Parents: d5603ae
Author: Rui Li <rui.li@intel.com>
Authored: Fri Jan 15 21:24:39 2016 +0800
Committer: Rui Li <rui.li@intel.com>
Committed: Fri Jan 15 21:25:02 2016 +0800

----------------------------------------------------------------------
 data/conf/spark/yarn-client/hive-site.xml          | 17 ++++++++++++++++-
 pom.xml                                            |  2 +-
 .../apache/hadoop/hive/shims/Hadoop23Shims.java    |  2 ++
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/814bdaa5/data/conf/spark/yarn-client/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml
index 1617976..ea6fdf4 100644
--- a/data/conf/spark/yarn-client/hive-site.xml
+++ b/data/conf/spark/yarn-client/hive-site.xml
@@ -241,7 +241,22 @@
 
 <property>
   <name>spark.executor.memory</name>
-  <value>512m</value>
+  <value>1g</value>
+</property>
+
+<property>
+  <name>spark.yarn.executor.memoryOverhead</name>
+  <value>0</value>
+</property>
+
+<property>
+  <name>spark.driver.memory</name>
+  <value>1g</value>
+</property>
+
+<property>
+  <name>spark.yarn.driver.memoryOverhead</name>
+  <value>0</value>
 </property>
 
 <property>

http://git-wip-us.apache.org/repos/asf/hive/blob/814bdaa5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 22b4ca8..be90d84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -162,7 +162,7 @@
     <ST4.version>4.0.4</ST4.version>
     <tez.version>0.8.1-alpha</tez.version>
     <super-csv.version>2.2.0</super-csv.version>
-    <spark.version>1.5.0</spark.version>
+    <spark.version>1.6.0</spark.version>
     <scala.binary.version>2.10</scala.binary.version>
     <scala.version>2.10.4</scala.version>
     <tempus-fugit.version>1.1</tempus-fugit.version>

http://git-wip-us.apache.org/repos/asf/hive/blob/814bdaa5/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 0b5c092..0670bc3 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -526,6 +526,8 @@ public class Hadoop23Shims extends HadoopShimsSecure {
       mr = new MiniSparkOnYARNCluster("sparkOnYarn");
       conf.set("fs.defaultFS", nameNode);
       conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
+      // disable resource monitoring, although it should be off by default
+      conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
       configureImpersonation(conf);
       mr.init(conf);
       mr.start();


Mime
View raw message