pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1209652 - in /pig/branches/branch-0.10: CHANGES.txt src/org/apache/pig/tools/pigstats/JobStats.java test/org/apache/pig/test/TestPigRunner.java
Date Fri, 02 Dec 2011 19:45:35 GMT
Author: daijy
Date: Fri Dec  2 19:45:35 2011
New Revision: 1209652

URL: http://svn.apache.org/viewvc?rev=1209652&view=rev
Log:
PIG-2358: JobStats.getHadoopCounters() is never set and always returns null

Modified:
    pig/branches/branch-0.10/CHANGES.txt
    pig/branches/branch-0.10/src/org/apache/pig/tools/pigstats/JobStats.java
    pig/branches/branch-0.10/test/org/apache/pig/test/TestPigRunner.java

Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1209652&r1=1209651&r2=1209652&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Fri Dec  2 19:45:35 2011
@@ -156,6 +156,8 @@ PIG-2228: support partial aggregation in
 
 BUG FIXES
 
+PIG-2358: JobStats.getHadoopCounters() is never set and always returns null (xutingz via
daijy)
+
 PIG-2384: PIG-2384: Generic Invokers should use PigContext to resolve classes (dvryaboy)
 
 PIG-2379: Bug in Schema.getPigSchema(ResourceSchema rSchema) improperly adds two level access
(jcoveney via dvryaboy)

Modified: pig/branches/branch-0.10/src/org/apache/pig/tools/pigstats/JobStats.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/tools/pigstats/JobStats.java?rev=1209652&r1=1209651&r2=1209652&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/tools/pigstats/JobStats.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/tools/pigstats/JobStats.java Fri Dec  2 19:45:35
2011
@@ -332,7 +332,6 @@ public final class JobStats extends Oper
 
     @SuppressWarnings("deprecation")
     void addCounters(RunningJob rjob) {
-        Counters counters = null;
         if (rjob != null) {
             try {
                 counters = rjob.getCounters();

Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestPigRunner.java?rev=1209652&r1=1209651&r2=1209652&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestPigRunner.java Fri Dec  2 19:45:35
2011
@@ -35,6 +35,7 @@ import junit.framework.Assert;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigRunner;
 import org.apache.pig.PigRunner.ReturnCode;
@@ -772,6 +773,44 @@ public class TestPigRunner {
         }
     }
     
+    @Test             //Pig-2358
+    public void testGetHadoopCounters() throws Exception {
+        final String OUTPUT_FILE_2 = "output2";
+        
+        PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
+        w.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+        w.println("B = filter A by a0 >= 4;");
+        w.println("C = filter A by a0 < 4;");
+        w.println("D = group C by a0;");
+        w.println("E = foreach D generate group, COUNT(C);");
+        w.println("store B into '" + OUTPUT_FILE_2 + "';");
+        w.println("store E into '" + OUTPUT_FILE + "';");
+        w.close();
+        
+        try {
+            String[] args = { PIG_FILE };
+            PigStats stats = PigRunner.run(args, new TestNotificationListener());
+            
+            Counters counter= ((JobStats)stats.getJobGraph().getSinks().get(0)).getHadoopCounters();
+            assertEquals(5, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+                    PigStatsUtil.MAP_INPUT_RECORDS).getValue());
+            assertEquals(3, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+                    PigStatsUtil.MAP_OUTPUT_RECORDS).getValue());
+            assertEquals(2, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+                    PigStatsUtil.REDUCE_INPUT_RECORDS).getValue());
+            assertEquals(0, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+                    PigStatsUtil.REDUCE_OUTPUT_RECORDS).getValue());
+            assertEquals(20,counter.getGroup(PigStatsUtil.FS_COUNTER_GROUP).getCounterForName(
+            		PigStatsUtil.HDFS_BYTES_WRITTEN).getValue());
+            assertEquals(30,counter.getGroup(PigStatsUtil.FS_COUNTER_GROUP).getCounterForName(
+            		PigStatsUtil.HDFS_BYTES_READ).getValue());
+        } finally {
+            new File(PIG_FILE).delete();
+            Util.deleteFile(cluster, OUTPUT_FILE);
+            Util.deleteFile(cluster, OUTPUT_FILE_2);
+        }
+    }
+    
     @Test // PIG-2208: Restrict number of PIG generated Haddop counters
     public void testDisablePigCounters2() throws Exception {
         



Mime
View raw message