pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billgra...@apache.org
Subject svn commit: r1397399 - in /pig/trunk: CHANGES.txt src/org/apache/pig/tools/pigstats/PigStats.java test/org/apache/pig/test/TestPigStats.java
Date Fri, 12 Oct 2012 00:47:56 GMT
Author: billgraham
Date: Fri Oct 12 00:47:55 2012
New Revision: 1397399

URL: http://svn.apache.org/viewvc?rev=1397399&view=rev
Log:
PIG-2964: Add helper method getJobList() to PigStats.JobGraph. Extend visibility of couple
methods on same class (prkommireddi via billgraham)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
    pig/trunk/test/org/apache/pig/test/TestPigStats.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Oct 12 00:47:55 2012
@@ -25,6 +25,8 @@ PIG-1891 Enable StoreFunc to make intell
 
 IMPROVEMENTS
 
+PIG-2964: Add helper method getJobList() to PigStats.JobGraph. Extend visibility of couple
methods on same class (prkommireddi via billgraham)
+
 PIG-2579: Support for multiple input schemas in AvroStorage (cheolsoo via sms)
 
 PIG-2946: Documentation of "history" and "clear" commands (xalan via azaroth)

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Fri Oct 12 00:47:55 2012
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.commons.collections.IteratorUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.JobClient;
@@ -216,6 +217,17 @@ public abstract class PigStats {
             return jp.toString();
         }
         
+        /**
+         * Returns a List representation of the Job graph. Returned list is an
+         * ArrayList
+         * 
+         * @return
+         */
+        @SuppressWarnings("unchecked")
+        public List<JobStats> getJobList() {
+            return IteratorUtils.toList(iterator());
+        }
+
         public Iterator<JobStats> iterator() {
             return new Iterator<JobStats>() {
                 private Iterator<Operator> iter = getOperators();                
@@ -246,7 +258,7 @@ public abstract class PigStats {
             return false;
         }
         
-        List<JobStats> getSuccessfulJobs() {
+        public List<JobStats> getSuccessfulJobs() {
             ArrayList<JobStats> lst = new ArrayList<JobStats>();
             Iterator<JobStats> iter = iterator();
             while (iter.hasNext()) {
@@ -259,7 +271,7 @@ public abstract class PigStats {
             return lst;
         }
         
-        List<JobStats> getFailedJobs() {
+        public List<JobStats> getFailedJobs() {
             ArrayList<JobStats> lst = new ArrayList<JobStats>();
             Iterator<JobStats> iter = iterator();
             while (iter.hasNext()) {

Modified: pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStats.java Fri Oct 12 00:47:55 2012
@@ -29,6 +29,9 @@ import java.io.PrintWriter;
 import junit.framework.Assert;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
@@ -41,11 +44,14 @@ import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.PigStats.JobGraph;
 import org.apache.pig.tools.pigstats.ScriptState;
 import org.junit.Test;
 
 public class TestPigStats  {
 
+    private static final Log LOG = LogFactory.getLog(TestPigStats.class);
+
     @Test
     public void testPigScriptInConf() throws Exception {
         PrintWriter w = new PrintWriter(new FileWriter("test.pig"));
@@ -123,7 +129,8 @@ public class TestPigStats  {
 
         File outputFile = null;
         try {
-            outputFile = File.createTempFile("JIAR_1027", ".out");
+            String fileName = this.getClass().getName() + "_" + "testBytesWritten_JIRA_1027";
+            outputFile = File.createTempFile(fileName, ".out");
             String filePath = outputFile.getAbsolutePath();
             outputFile.delete();
             PigServer pig = new PigServer(ExecType.LOCAL);
@@ -133,9 +140,8 @@ public class TestPigStats  {
             File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000"
);
             assertEquals(dataFile.length(), stats.getBytesWritten());
         } catch (IOException e) {
-            e.printStackTrace();
-            System.err.println( e.getMessage() );
-            fail("IOException happened");
+            LOG.error("Error while generating file", e);
+            fail("Encountered IOException");
         } finally {
             if (outputFile != null) {
                 // Hadoop Local mode creates a directory
@@ -181,18 +187,43 @@ public class TestPigStats  {
         }
     }
     
-    private void deleteDirectory( File dir ) {
-        File[] files = dir.listFiles();
-        for( File file : files ) {
-            if( file.isDirectory() ) {
-                deleteDirectory(file);
-            } else {
-                file.delete();
+    @Test
+    public void testPigStatsGetList() {
+        File outputFile = null;
+        try {
+            String filename = this.getClass().getSimpleName() + "_" + "testPigStatsGetList";
+            outputFile = File.createTempFile(filename, ".out");
+            String filePath = outputFile.getAbsolutePath();
+            outputFile.delete();
+            PigServer pigServer = new PigServer(ExecType.LOCAL);
+            pigServer.registerQuery("a = load 'test/org/apache/pig/test/data/passwd';");
+            pigServer.registerQuery("b = group a by $0;");
+            pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
+            pigServer.registerQuery("d = group c by cnt;");
+            pigServer.registerQuery("e = foreach d generate group;");
+            ExecJob job = pigServer.store("e", filePath);
+            JobGraph jobGraph = job.getStatistics().getJobGraph();
+            assertEquals(2, jobGraph.getJobList().size());
+
+        } catch (IOException e) {
+            LOG.error("IOException while creating file ", e);
+            fail("Encountered IOException");
+        } finally {
+            if (outputFile != null) {
+                // delete the directory before returning
+                deleteDirectory(outputFile);
             }
         }
-        dir.delete();
     }
     
+    private void deleteDirectory(File dir) {
+        try {
+            FileUtils.deleteDirectory(dir);
+        } catch (IOException e) {
+            LOG.error("Could not delete directory " + dir, e);
+        }
+    }
+
     public static LogicalPlan getLogicalPlan(PigServer pig) throws Exception {
         java.lang.reflect.Method buildLp = pig.getClass().getDeclaredMethod("buildLp");
         buildLp.setAccessible(true);



Mime
View raw message