pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1751389 - in /pig/branches/spark: src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java test/org/apache/pig/test/TestFinish.java
Date Tue, 05 Jul 2016 03:46:51 GMT
Author: xuefu
Date: Tue Jul  5 03:46:51 2016
New Revision: 1751389

URL: http://svn.apache.org/viewvc?rev=1751389&view=rev
Log:
PIG-4281: Fix TestFinish for Spark engine (Liyun via Xuefu)

Modified:
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
    pig/branches/spark/test/org/apache/pig/test/TestFinish.java

Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1751389&r1=1751388&r2=1751389&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Tue Jul  5 03:46:51 2016
@@ -47,6 +47,7 @@ import org.apache.pig.backend.BackendExc
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.backend.hadoop.executionengine.Launcher;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
@@ -650,5 +651,6 @@ public class SparkLauncher extends Launc
 
         SchemaTupleBackend.initialize(jobConf, pigContext);
         Utils.setDefaultTimeZone(jobConf);
+        PigMapReduce.sJobConfInternal.set(jobConf);
     }
 }

Modified: pig/branches/spark/test/org/apache/pig/test/TestFinish.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestFinish.java?rev=1751389&r1=1751388&r2=1751389&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestFinish.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestFinish.java Tue Jul  5 03:46:51 2016
@@ -45,7 +45,7 @@ public class TestFinish {
     BagFactory mBf = BagFactory.getInstance();
     File f1;
 
-    static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+    private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
 
     static public class MyEvalFunction extends EvalFunc<Tuple> {
         String execType;
@@ -67,12 +67,7 @@ public class TestFinish {
         @Override
         public void finish() {
             try {
-                FileSystem fs = null;
-                if (execType.equalsIgnoreCase("SPARK")) {
-                    fs = FileSystem.get(cluster.getConfiguration());
-                } else {
-                    fs = FileSystem.get(PigMapReduce.sJobConfInternal.get());
-                }
+                FileSystem fs = FileSystem.get(PigMapReduce.sJobConfInternal.get());
                 fs.create(new Path(expectedFileName));
             } catch (IOException e) {
                 throw new RuntimeException("Unable to create file:" + expectedFileName);
@@ -141,7 +136,7 @@ public class TestFinish {
         String inputFileName = setUp(cluster.getExecType());
         // this file will be created on the cluster if finish() is called
         String expectedFileName = "testFinishInMapMR-finish.txt";
-        pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('"+cluster.getExecType()+"','"
+        pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
                 + expectedFileName + "');");
         pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using
"
                 + PigStorage.class.getName() + "(':');");
@@ -160,7 +155,7 @@ public class TestFinish {
         String inputFileName = setUp(cluster.getExecType());
         // this file will be created on the cluster if finish() is called
         String expectedFileName = "testFinishInReduceMR-finish.txt";
-        pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('"+cluster.getExecType()+"','"
+        pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
                 + expectedFileName + "');");
         pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using
"
                 + PigStorage.class.getName() + "(':');");



Mime
View raw message