pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r903030 - in /hadoop/pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/impl/util/ test/org/apache/pig/test/ test/org/apache/pig/test/data/GoldenFiles/ test/org/apache/pig/test/utils/
Date Tue, 26 Jan 2010 00:34:14 GMT
Author: daijy
Date: Tue Jan 26 00:34:14 2010
New Revision: 903030

URL: http://svn.apache.org/viewvc?rev=903030&view=rev
Log:
PIG-1189: StoreFunc UDF should ship to the backend automatically without "register"

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
    hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
    hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
    hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
    hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue Jan 26 00:34:14 2010
@@ -147,6 +147,9 @@
 PIG-1184: PruneColumns optimization does not handle the case of foreach
 flatten correctly if flattened bag is not used later (daijy)
 
+PIG-1189: StoreFunc UDF should ship to the backend automatically without
+"register" (daijy)
+
 Release 0.6.0 - Unreleased
 
 INCOMPATIBLE CHANGES

Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Tue Jan 26 00:34:14 2010
@@ -732,6 +732,8 @@
             storeToMapReduceMap.put(op, curMROp);
             nonBlocking(op);
             phyToMROpMap.put(op, curMROp);
+            if (op.getSFile()!=null && op.getSFile().getFuncSpec()!=null)
+                curMROp.UDFs.add(op.getSFile().getFuncSpec().toString());
         }catch(Exception e){
             int errCode = 2034;
             String msg = "Error compiling operator " + op.getClass().getSimpleName();

Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
Tue Jan 26 00:34:14 2010
@@ -20,8 +20,10 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
+import java.util.Set;
 
 import org.apache.pig.impl.io.FileSpec;
 import org.apache.pig.impl.plan.OperatorKey;
@@ -105,7 +107,7 @@
     // Sort order for secondary keys;
     boolean[] secondarySortOrder;
 
-    public List<String> UDFs;
+    public Set<String> UDFs;
     
     // Indicates if a UDF comparator is used
     boolean isUDFComparatorUsed = false;
@@ -142,7 +144,7 @@
         mapPlan = new PhysicalPlan();
         combinePlan = new PhysicalPlan();
         reducePlan = new PhysicalPlan();
-        UDFs = new ArrayList<String>();
+        UDFs = new HashSet<String>();
         nig = NodeIdGenerator.getGenerator();
         scope = k.getScope();
     }

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Tue Jan 26 00:34:14 2010
@@ -32,6 +32,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Vector;
 import java.util.jar.JarEntry;
 import java.util.jar.JarInputStream;
@@ -96,7 +97,7 @@
      * @throws ClassNotFoundException
      * @throws IOException
      */
-    public static void createJar(OutputStream os, List<String> funcs, PigContext pigContext)
throws ClassNotFoundException, IOException {
+    public static void createJar(OutputStream os, Set<String> funcs, PigContext pigContext)
throws ClassNotFoundException, IOException {
         Vector<JarListEntry> jarList = new Vector<JarListEntry>();
         for(String toSend: pigPackagesToSend) {
             addContainingJar(jarList, PigMapReduce.class, toSend, pigContext);

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Tue Jan 26 00:34:14 2010
@@ -914,8 +914,9 @@
         MapReduceOper mrOper = mrPlan.getRoots().get(0);
         
         assertTrue(mrOper.UDFs.size()==2);
-        assertTrue(mrOper.UDFs.get(0).equals("BinStorage"));
-        assertTrue(mrOper.UDFs.get(1).equals("org.apache.pig.builtin.PigStorage"));
+        assertTrue(mrOper.UDFs.size()==2);
+        assertTrue(mrOper.UDFs.contains("BinStorage"));
+        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));
     }
 
     @Test

Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Tue Jan 26 00:34:14
2010
@@ -26,7 +26,7 @@
     |   |
     |   |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage)
- -152
     |
-    |---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -145:
+    |---MapReduce(20,SUM,COUNT,TestMRCompiler$WeirdComparator) - -145:
         |   Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage)
- -151
         |   |
         |   |---New For Each(false,false)[tuple] - -2197807331204639125
@@ -88,4 +88,4 @@
                 |       |   |
                 |       |   Constant(true) - --6860857982727545176
                 |       |
-                |       |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
+                |       |---Load(DummyFil:DummyLdr) - -737965411848544085

Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java Tue Jan 26 00:34:14 2010
@@ -762,7 +762,6 @@
 
     public static POStore topStoreOp() {
         POStore ret = new POStore(new OperatorKey("", r.nextLong()));
-        ret.setSFile(new FileSpec("DummyFil", new FuncSpec("DummyLdr")));
         return ret;
     }
 



Mime
View raw message