pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1603547 - in /pig/trunk: CHANGES.txt src/org/apache/pig/PigConfiguration.java src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
Date Wed, 18 Jun 2014 17:36:10 GMT
Author: cheolsoo
Date: Wed Jun 18 17:36:10 2014
New Revision: 1603547

URL: http://svn.apache.org/r1603547
Log:
PIG-4006: Make the interval of DAGStatus report configurable (cheolsoo)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/PigConfiguration.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 18 17:36:10 2014
@@ -76,6 +76,8 @@ PIG-2207: Support custom counters for ag
 
 IMPROVEMENTS
 
+PIG-4006: Make the interval of DAGStatus report configurable (cheolsoo)
+
 PIG-3999: Document PIG-3388 (lbendig via cheolsoo)
 
 PIG-3954: Document use of user level jar cache (aniket486)

Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Wed Jun 18 17:36:10 2014
@@ -92,6 +92,11 @@ public class PigConfiguration {
     public static final String TEZ_SESSION_REUSE = "pig.tez.session.reuse";
 
     /**
+     * This key is used to configure the interval of dag status report in seconds.
+     */
+    public static final String TEZ_DAG_STATUS_REPORT_INTERVAL = "pig.tez.dag.status.report.interval";
+
+    /**
      * Turns off use of combiners in MapReduce jobs produced by Pig.
      */
     public static final String PROP_NO_COMBINER = "pig.exec.nocombiner";

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java Wed Jun 18
17:36:10 2014
@@ -22,6 +22,8 @@ import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -64,6 +66,8 @@ public class TezJob extends ControlledJo
     private TezCounters dagCounters;
     // Vertex, CounterGroup, Counter, Value
     private Map<String, Map<String, Map<String, Long>>> vertexCounters;
+    // Timer for DAG status reporter
+    private Timer timer;
 
     public TezJob(TezConfiguration conf, DAG dag, Map<String, LocalResource> requestAMResources)
             throws IOException {
@@ -131,6 +135,10 @@ public class TezJob extends ControlledJo
             return;
         }
 
+        timer = new Timer();
+        timer.schedule(new DAGStatusReporter(), 1000, conf.getLong(
+                PigConfiguration.TEZ_DAG_STATUS_REPORT_INTERVAL, 10) * 1000);
+
         while (true) {
             try {
                 dagStatus = dagClient.getDAGStatus(statusGetOpts);
@@ -140,7 +148,6 @@ public class TezJob extends ControlledJo
                 break;
             }
 
-            log.info("DAG Status: " + dagStatus);
             setJobState(dagState2JobState(dagStatus.getState()));
             if (dagStatus.isCompleted()) {
                 StringBuilder sb = new StringBuilder();
@@ -171,6 +178,14 @@ public class TezJob extends ControlledJo
                 // Do nothing
             }
         }
+
+        timer.cancel();
+    }
+
+    private class DAGStatusReporter extends TimerTask {
+        public void run() {
+            log.info("DAG Status: " + dagStatus);
+        }
     }
 
     private void collectVertexCounters() {



Mime
View raw message