pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject svn commit: r1787682 - in /pig/branches/spark/src/org/apache/pig: backend/hadoop/executionengine/spark/SparkLauncher.java impl/builtin/StreamingUDF.java scripting/groovy/GroovyEvalFunc.java
Date Mon, 20 Mar 2017 02:55:24 GMT
Author: zly
Date: Mon Mar 20 02:55:23 2017
New Revision: 1787682

URL: http://svn.apache.org/viewvc?rev=1787682&view=rev
Log:
PIG-5177:Scripting and StreamingPythonUDFs fail with Spark exec type (Adam via Liyun)

Modified:
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
    pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java
    pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java

Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Mon Mar 20 02:55:23 2017
@@ -417,6 +417,12 @@ public class SparkLauncher extends Launc
             allJars.add(udfJar);
         }
 
+        File scriptUDFJarFile = JarManager.createPigScriptUDFJar(pigContext);
+        if (scriptUDFJarFile != null) {
+            LOG.info("add script udf jar to Spark job");
+            allJars.add(scriptUDFJarFile.getAbsolutePath().toString());
+        }
+
         //Upload all jars to spark working directory
         for (String jar : allJars) {
             File jarFile = new File(jar);

Modified: pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java (original)
+++ pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java Mon Mar 20 02:55:23
2017
@@ -242,8 +242,11 @@ public class StreamingUDF extends EvalFu
             }
             InputStream udfFileStream = this.getClass().getResourceAsStream(
                     absolutePath + getUserFileExtension());
-            command[PATH_TO_FILE_CACHE] = "\"" + userUdfFile.getParentFile().getAbsolutePath()
-                    + "\"";
+            if (udfFileStream == null) {
+                //Try loading the script from other locally available jars (needed for Spark
mode)
+                udfFileStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath+getUserFileExtension());
+            }
+            command[PATH_TO_FILE_CACHE] = "\"" + userUdfFile.getParentFile().getAbsolutePath()
+ "\"";
 
             try {
                 FileUtils.copyInputStreamToFile(udfFileStream, userUdfFile);

Modified: pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java (original)
+++ pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java Mon Mar 20
02:55:23 2017
@@ -74,6 +74,10 @@ public class GroovyEvalFunc<T> extends E
               resource = ScriptEngine.class.getResource(File.separator + path);
           }
           if (resource == null) {
+            //Try loading the script from other locally available jars (needed for Spark
mode)
+              resource = Thread.currentThread().getContextClassLoader().getResource(path);
+          }
+          if (resource == null) {
               throw new IOException("Cannot find " + path);
           }
       } else {



Mime
View raw message