pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject svn commit: r1791059 - in /pig/branches/spark/src/org/apache/pig: PigConfiguration.java backend/hadoop/executionengine/spark/SparkLauncher.java
Date Wed, 12 Apr 2017 01:52:15 GMT
Author: zly
Date: Wed Apr 12 01:52:15 2017
New Revision: 1791059

URL: http://svn.apache.org/viewvc?rev=1791059&view=rev
Log:
PIG-5176:Several ComputeSpec test cases fail(Nandor via Liyun)

Modified:
    pig/branches/spark/src/org/apache/pig/PigConfiguration.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java

Modified: pig/branches/spark/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/PigConfiguration.java?rev=1791059&r1=1791058&r2=1791059&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/branches/spark/src/org/apache/pig/PigConfiguration.java Wed Apr 12 01:52:15 2017
@@ -482,6 +482,11 @@ public class PigConfiguration {
     public static final String PIG_LOG_TRACE_ID = "pig.log.trace.id";
 
     /**
+     * Use Netty file server for Pig on Spark, true or false, default value is false
+     */
+    public static final String PIG_SPARK_USE_NETTY_FILESERVER = "pig.spark.rpc.useNettyFileServer";
+
+    /**
      * @deprecated use {@link #PIG_LOG_TRACE_ID} instead. Will be removed in Pig 0.18
      */
     public static final String CALLER_ID = PIG_LOG_TRACE_ID;

Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1791059&r1=1791058&r2=1791059&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Wed Apr 12 01:52:15 2017
@@ -559,6 +559,12 @@ public class SparkLauncher extends Launc
 
             sparkConf.setMaster(master);
             sparkConf.setAppName("PigOnSpark:" + pigCtxtProperties.getProperty(PigContext.JOB_NAME));
+            // On Spark 1.6, Netty file server doesn't allow adding the same file with the
same name twice
+            // This is a problem for streaming using a script + explicit ship the same script
combination (PIG-5134)
+            // HTTP file server doesn't have this restriction, it overwrites the file if
added twice
+            String useNettyFileServer = pigCtxtProperties.getProperty(PigConfiguration.PIG_SPARK_USE_NETTY_FILESERVER,
"false");
+            sparkConf.set("spark.rpc.useNettyFileServer", useNettyFileServer);
+
             if (sparkHome != null && !sparkHome.isEmpty()) {
                 sparkConf.setSparkHome(sparkHome);
             } else {



Mime
View raw message