pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r904710 - in /hadoop/pig/branches/branch-0.6: CHANGES.txt src/org/apache/pig/impl/logicalLayer/LOForEach.java test/org/apache/pig/test/TestPruneColumn.java
Date Sat, 30 Jan 2010 02:19:27 GMT
Author: daijy
Date: Sat Jan 30 02:19:27 2010
New Revision: 904710

URL: http://svn.apache.org/viewvc?rev=904710&view=rev
Log:
PIG-1210: fieldsToRead send the same fields more than once in some cases

Modified:
    hadoop/pig/branches/branch-0.6/CHANGES.txt
    hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
    hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java

Modified: hadoop/pig/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/CHANGES.txt?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/CHANGES.txt Sat Jan 30 02:19:27 2010
@@ -281,6 +281,8 @@
 
 PIG-1195: POSort should take care of sort order (daijy)
 
+PIG-1210: fieldsToRead send the same fields more than once in some cases (daijy)
+
 Release 0.5.0
 
 INCOMPATIBLE CHANGES

Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java Sat
Jan 30 02:19:27 2010
@@ -866,7 +866,8 @@
         ArrayList<Pair<Integer, Integer>> inputList = new ArrayList<Pair<Integer,
Integer>>();
         for (LOProject project : projectFinder.getProjectSet()) {
             for (int inputColumn : project.getProjection()) {
-                inputList.add(new Pair<Integer, Integer>(0, inputColumn));
+                if (!inputList.contains(new Pair<Integer, Integer>(0, inputColumn)))
+                    inputList.add(new Pair<Integer, Integer>(0, inputColumn));
             }
         }
         if (inputList.size()==0)

Modified: hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java Sat Jan 30
02:19:27 2010
@@ -33,8 +33,12 @@
 import org.apache.log4j.SimpleLayout;
 import org.apache.pig.ExecType;
 import org.apache.pig.FilterFunc;
+import org.apache.pig.LoadFunc;
 import org.apache.pig.PigServer;
+import org.apache.pig.LoadFunc.RequiredFieldList;
+import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,8 +59,24 @@
     File tmpFile9;
     File tmpFile10;
     File logFile;
-    Logger logger;
 
+    static public class PigStorageWithTrace extends PigStorage {
+
+        /**
+         * @param delimiter
+         */
+        public PigStorageWithTrace() {
+            super();
+        }
+        @Override
+        public LoadFunc.RequiredFieldResponse fieldsToRead(RequiredFieldList requiredFieldList)
throws FrontendException {
+            LoadFunc.RequiredFieldResponse response = super.fieldsToRead(requiredFieldList);
+            Logger logger = Logger.getLogger(this.getClass());
+            logger.info(requiredFieldList);
+            return response;
+        }
+
+    }
     private static final String simpleEchoStreamingCommand;
     static {
         if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
@@ -76,7 +96,7 @@
     @Before
     @Override
     public void setUp() throws Exception{
-        logger = Logger.getLogger(PruneColumns.class);
+        Logger logger = Logger.getLogger(PruneColumns.class);
         logger.removeAllAppenders();
         logger.setLevel(Level.INFO);
         SimpleLayout layout = new SimpleLayout();
@@ -84,6 +104,10 @@
         FileAppender appender = new FileAppender(layout, logFile.toString(), false, false,
0);
         logger.addAppender(appender);
         
+        Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class);
+        pigStorageWithTraceLogger.setLevel(Level.INFO);
+        pigStorageWithTraceLogger.addAppender(appender);
+        
         pigServer = new PigServer("local");
         //pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
         tmpFile1 = File.createTempFile("prune", "txt");
@@ -1673,4 +1697,27 @@
         assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0, $2", 
             "No map keys pruned for A", "No column pruned for B", "No map keys pruned for
B"}));
     }
+    
+    // See PIG-1210
+    @Test
+    public void testFieldsToReadDuplicatedEntry() throws Exception {
+        pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "'
using "+PigStorageWithTrace.class.getName()
+                +" AS (a0, a1, a2);");
+        pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;");
+        Iterator<Tuple> iter = pigServer.openIterator("B");
+
+        assertTrue(iter.hasNext());
+        Tuple t = iter.next();
+        assertTrue(t.toString().equals("(2.0,2,3)"));
+        
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertTrue(t.toString().equals("(4.0,5,2)"));
+
+        assertFalse(iter.hasNext());
+
+        assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+            "No map keys pruned for A", "[0,1,2]"}));
+    }
+
 }



Mime
View raw message