pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dvrya...@apache.org
Subject svn commit: r1376819 - in /pig/trunk: ./ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/ shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ t...
Date Fri, 24 Aug 2012 03:37:50 GMT
Author: dvryaboy
Date: Fri Aug 24 03:37:50 2012
New Revision: 1376819

URL: http://svn.apache.org/viewvc?rev=1376819&view=rev
Log:
PIG-2890: Revert PIG-2578

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
    pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
    pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 03:37:50 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2890: Revert PIG-2578 (dvryaboy)
+
 PIG-2850: Pig should support loading macro files as resources stored in JAR files (matterhayes
via dvryaboy)
 
 PIG-1314: Add DateTime Support to Pig (zjshen via thejas)

Modified: pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
(original)
+++ pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
Fri Aug 24 03:37:50 2012
@@ -50,7 +50,7 @@ public class HadoopShims {
 
     static public TaskAttemptContext createTaskAttemptContext(Configuration conf,
                                 TaskAttemptID taskId) {
-        TaskAttemptContext newContext = new TaskAttemptContext(new Configuration(conf),
+        TaskAttemptContext newContext = new TaskAttemptContext(conf,
             taskId);
         return newContext;
     }
@@ -58,7 +58,7 @@ public class HadoopShims {
     static public JobContext createJobContext(Configuration conf,
             JobID jobId) {
         JobContext newJobContext = new JobContext(
-                new Configuration(conf), jobId);
+                conf, jobId);
         return newJobContext;
     }
 

Modified: pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
(original)
+++ pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
Fri Aug 24 03:37:50 2012
@@ -43,13 +43,13 @@ public class HadoopShims {
 
     static public TaskAttemptContext createTaskAttemptContext(Configuration conf,
                                 TaskAttemptID taskId) {
-        TaskAttemptContext newContext = new TaskAttemptContextImpl(new Configuration(conf),
taskId);
+        TaskAttemptContext newContext = new TaskAttemptContextImpl(conf, taskId);
         return newContext;
     }
 
     static public JobContext createJobContext(Configuration conf,
             JobID jobId) {
-        JobContext newContext = new JobContextImpl(new Configuration(conf), jobId);
+        JobContext newContext = new JobContextImpl(conf, jobId);
         return newContext;
     }
 

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Fri Aug 24 03:37:50 2012
@@ -498,13 +498,13 @@ public class JobControlCompiler{
             for (POStore st: mapStores) {
                 storeLocations.add(st);
                 StoreFuncInterface sFunc = st.getStoreFunc();
-                sFunc.setStoreLocation(st.getSFile().getFileName(), new org.apache.hadoop.mapreduce.Job(nwJob.getConfiguration()));
+                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
             }
 
             for (POStore st: reduceStores) {
                 storeLocations.add(st);
                 StoreFuncInterface sFunc = st.getStoreFunc();
-                sFunc.setStoreLocation(st.getSFile().getFileName(), new org.apache.hadoop.mapreduce.Job(nwJob.getConfiguration()));
+                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
             }
 
             // the OutputFormat we report to Hadoop is always PigOutputFormat

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
Fri Aug 24 03:37:50 2012
@@ -84,6 +84,7 @@ import org.apache.pig.impl.io.NullableDo
 import org.apache.pig.impl.io.NullableFloatWritable;
 import org.apache.pig.impl.io.NullableIntWritable;
 import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
 import org.apache.pig.impl.io.NullablePartitionWritable;
 import org.apache.pig.impl.io.NullableText;
 import org.apache.pig.impl.io.NullableTuple;
@@ -956,6 +957,12 @@ public class JobControlCompiler{
         }
     }
 
+    public static class PigDateTimeWritableComparator extends PigWritableComparator {
+        public PigDateTimeWritableComparator() {
+            super(NullableDateTimeWritable.class);
+        }
+    }
+
     public static class PigCharArrayWritableComparator extends PigWritableComparator {
         public PigCharArrayWritableComparator() {
             super(NullableText.class);
@@ -1012,6 +1019,12 @@ public class JobControlCompiler{
         }
     }
 
+    public static class PigGroupingDateTimeWritableComparator extends WritableComparator
{
+        public PigGroupingDateTimeWritableComparator() {
+            super(NullableDateTimeWritable.class, true);
+        }
+    }
+
     public static class PigGroupingCharArrayWritableComparator extends WritableComparator
{
         public PigGroupingCharArrayWritableComparator() {
             super(NullableText.class, true);
@@ -1087,6 +1100,10 @@ public class JobControlCompiler{
                 job.setSortComparatorClass(PigDoubleRawComparator.class);
                 break;
 
+            case DataType.DATETIME:
+                job.setSortComparatorClass(PigDateTimeRawComparator.class);
+                break;
+
             case DataType.CHARARRAY:
                 job.setSortComparatorClass(PigTextRawComparator.class);
                 break;
@@ -1141,6 +1158,11 @@ public class JobControlCompiler{
             job.setGroupingComparatorClass(PigGroupingDoubleWritableComparator.class);
             break;
 
+        case DataType.DATETIME:
+            job.setSortComparatorClass(PigDateTimeWritableComparator.class);
+            job.setGroupingComparatorClass(PigGroupingDateTimeWritableComparator.class);
+            break;
+
         case DataType.CHARARRAY:
             job.setSortComparatorClass(PigCharArrayWritableComparator.class);
             job.setGroupingComparatorClass(PigGroupingCharArrayWritableComparator.class);

Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java Fri Aug 24 03:37:50 2012
@@ -17,8 +17,6 @@
  */
 package org.apache.pig.test;
 
-import java.io.BufferedReader;
-import java.io.FileReader;
 import java.io.StringReader;
 import java.io.IOException;
 import java.io.File;
@@ -31,16 +29,12 @@ import java.util.Properties;
 import junit.framework.Assert;
 import junit.framework.TestCase;
 
-import org.apache.hadoop.mapreduce.Job;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigException;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecJob;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.plan.Operator;
 import org.apache.pig.impl.plan.OperatorPlan;
@@ -341,69 +335,6 @@ public class TestMultiQueryLocal {
             Assert.fail();
         }
     }
-    
-    public static class PigStorageWithSuffix extends PigStorage {
-
-        private String suffix;
-        public PigStorageWithSuffix(String s) {
-            this.suffix = s;
-        }
-        static private final String key="test.key";
-        @Override
-        public void setStoreLocation(String location, Job job) throws IOException {
-            super.setStoreLocation(location, job);
-            if (job.getConfiguration().get(key)==null) {
-                job.getConfiguration().set(key, suffix);
-            }
-            suffix = job.getConfiguration().get(key);
-        }
-        
-        @Override
-        public void putNext(Tuple f) throws IOException {
-            try {
-                Tuple t = TupleFactory.getInstance().newTuple();
-                for (Object obj : f.getAll()) {
-                    t.append(obj);
-                }
-                t.append(suffix);
-                writer.write(null, t);
-            } catch (InterruptedException e) {
-                throw new IOException(e);
-            }
-        }
-    }
-    
-    // See PIG-2578
-    @Test
-    public void testMultiStoreWithConfig() {
-
-        System.out.println("===== test multi-query with competing config =====");
-
-        try {
-            myPig.setBatchOn();
-
-            myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " +
-                                "using PigStorage(':') as (uname:chararray, passwd:chararray,
uid:int,gid:int);");
-            myPig.registerQuery("store a into '/tmp/Pig-TestMultiQueryLocal1' using " + PigStorageWithSuffix.class.getName()
+ "('a');");
-            myPig.registerQuery("store a into '/tmp/Pig-TestMultiQueryLocal2' using " + PigStorageWithSuffix.class.getName()
+ "('b');");
-
-            myPig.executeBatch();
-            myPig.discardBatch();
-            BufferedReader reader = new BufferedReader(new FileReader("/tmp/Pig-TestMultiQueryLocal1/part-m-00000"));
-            String line;
-            while ((line = reader.readLine())!=null) {
-                Assert.assertTrue(line.endsWith("a"));
-            }
-            reader = new BufferedReader(new FileReader("/tmp/Pig-TestMultiQueryLocal2/part-m-00000"));
-            while ((line = reader.readLine())!=null) {
-                Assert.assertTrue(line.endsWith("b"));
-            }
-
-        } catch (Exception e) {
-            e.printStackTrace();
-            Assert.fail();
-        }
-    }
 
     @Test
     public void testMultiQueryWithExplain() {



Mime
View raw message