hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1653202 - /hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Date Tue, 20 Jan 2015 06:30:13 GMT
Author: xuefu
Date: Tue Jan 20 06:30:12 2015
New Revision: 1653202

URL: http://svn.apache.org/r1653202
Log:
HIVE-9342: add num-executors / executor-cores / executor-memory option support for hive on
spark in Yarn mode [Spark Branch] (Pierre Yin via Xuefu)

Modified:
    hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1653202&r1=1653201&r2=1653202&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
(original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Tue Jan 20 06:30:12 2015
@@ -299,6 +299,25 @@ class SparkClientImpl implements SparkCl
         argv.add("org.apache.spark.deploy.SparkSubmit");
       }
 
+      if (master.equals("yarn-cluster")) {
+        String executorCores = conf.get("spark.executor.cores");
+        if (executorCores != null) {
+          argv.add("--executor-cores");
+          argv.add(executorCores);
+        }
+
+        String executorMemory = conf.get("spark.executor.memory");
+        if (executorMemory != null) {
+          argv.add("--executor-memory");
+          argv.add(executorMemory);
+        }
+
+        String numOfExecutors = conf.get("spark.executor.instances");
+        if (numOfExecutors != null) {
+          argv.add("--num-executors");
+          argv.add(numOfExecutors);
+        }
+      }
 
       argv.add("--properties-file");
       argv.add(properties.getAbsolutePath());



Mime
View raw message