pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1751849 - in /pig/branches/spark: src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java test/org/apache/pig/test/TestEvalPipelineLocal.java
Date Fri, 08 Jul 2016 02:38:18 GMT
Author: xuefu
Date: Fri Jul  8 02:38:18 2016
New Revision: 1751849

URL: http://svn.apache.org/viewvc?rev=1751849&view=rev
Log:
PIG-4944: Reset UDFContext#jobConf in spark mode (Liyun via Xuefu)

Modified:
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
    pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java

Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1751849&r1=1751848&r2=1751849&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Fri Jul  8 02:38:18 2016
@@ -224,9 +224,14 @@ public class SparkLauncher extends Launc
         new JobGraphBuilder(sparkplan, convertMap, sparkStats, sparkContext, jobMetricsListener,
jobGroupID, jobConf, pigContext).visit();
         cleanUpSparkJob(sparkStats);
         sparkStats.finish();
+        resetUDFContext();
         return sparkStats;
     }
 
+    private void resetUDFContext() {
+        UDFContext.getUDFContext().addJobConf(null);
+    }
+
     private void uploadResources(SparkOperPlan sparkPlan) throws IOException {
         addFilesToSparkJob();
         addJarsToSparkJob(sparkPlan);

Modified: pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1751849&r1=1751848&r2=1751849&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java Fri Jul  8 02:38:18
2016
@@ -65,7 +65,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 public class TestEvalPipelineLocal {
-    
+
     private PigServer pigServer;
 
     static final int MAX_SIZE = 100000;
@@ -1114,8 +1114,6 @@ public class TestEvalPipelineLocal {
     
     @Test
     public void testSetLocationCalledInFE() throws Exception {
-        // Need to reset it when running multiple testcases
-        UDFContext.getUDFContext().addJobConf(null);
         File f1 = createFile(new String[]{"a","b"});
         pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
                 + "' using " + SetLocationTestLoadFunc.class.getName()



Mime
View raw message