hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r998415 - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/exec/
Date Sat, 18 Sep 2010 06:33:01 GMT
Author: namit
Date: Sat Sep 18 06:33:01 2010
New Revision: 998415

URL: http://svn.apache.org/viewvc?rev=998415&view=rev
Log:
HIVE-1617. ScriptOperator's AutoProgressor can lead to an infinite loop
(Paul Yang via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/conf/hive-default.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Sat Sep 18 06:33:01 2010
@@ -132,6 +132,9 @@ Trunk -  Unreleased
     HIVE-1645. ability to specify parent directory for zookeeper lock manager
     (namit via He Yongqiang)
 
+    HIVE-1617. ScriptOperator's AutoProgressor can lead to an infinite loop
+    (Paul Yang via namit)
+
     HIVE-1226. support filter pushdown against non-native tables
     (jvs via He Yongqiang)
 

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Sep 18
06:33:01 2010
@@ -182,6 +182,7 @@ public class HiveConf extends Configurat
     HIVEADDEDARCHIVES("hive.added.archives.path", ""),
 
     // for hive script operator
+    HIVES_AUTO_PROGRESS_TIMEOUT("hive.auto.progress.timeout", 0),
     HIVETABLENAME("hive.table.name", ""),
     HIVEPARTITIONNAME("hive.partition.name", ""),
     HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", false),

Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Sat Sep 18 06:33:01 2010
@@ -695,6 +695,14 @@
   </description>
 </property>
 
+<property>
+  <name>hive.auto.progress.timeout</name>
+  <value>0</value>
+  <description> 
+    How long to run autoprogressor for the script/UDTF operators (in seconds). 
+    Set to 0 for forever.
+  </description>
+</property>
 
 <!-- HBase Storage Handler Parameters -->
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java Sat Sep
18 06:33:01 2010
@@ -28,7 +28,8 @@ import org.apache.hadoop.mapred.Reporter
 /**
  * AutoProgressor periodically sends updates to the job tracker so that it
  * doesn't consider this task attempt dead if there is a long period of
- * inactivity.
+ * inactivity. This can be configured with a timeout so that it doesn't run
+ * indefinitely.
  */
 public class AutoProgressor {
   protected Log LOG = LogFactory.getLog(this.getClass().getName());
@@ -38,9 +39,12 @@ public class AutoProgressor {
   // duration, a progress report is sent to the tracker so that the tracker
   // does not think that the job is dead.
   Timer rpTimer = null;
+  // Timer that tops rpTimer after a long timeout, e.g. 1 hr
+  Timer srpTimer = null;
   // Name of the class to report for
   String logClassName = null;
   int notificationInterval;
+  int timeout;
   Reporter reporter;
 
   class ReporterTask extends TimerTask {
@@ -70,18 +74,68 @@ public class AutoProgressor {
     }
   }
 
+  class StopReporterTimerTask extends TimerTask {
+
+    /**
+     * Task to stop the reporter timer once we hit the timeout
+     */
+    private final ReporterTask rt;
+
+    public StopReporterTimerTask(ReporterTask rp) {
+      this.rt = rp;
+    }
+
+    @Override
+    public void run() {
+      if (rt != null) {
+        LOG.info("Stopping reporter timer for " + logClassName);
+        rt.cancel();
+      }
+    }
+  }
+
+  /**
+   *
+   * @param logClassName
+   * @param reporter
+   * @param notificationInterval - interval for reporter updates (in ms)
+   */
   AutoProgressor(String logClassName, Reporter reporter,
       int notificationInterval) {
     this.logClassName = logClassName;
     this.reporter = reporter;
     this.notificationInterval = notificationInterval;
+    this.timeout = 0;
+  }
+
+  /**
+   *
+   * @param logClassName
+   * @param reporter
+   * @param notificationInterval - interval for reporter updates (in ms)
+   * @param timeout - when the autoprogressor should stop reporting (in ms)
+   */
+  AutoProgressor(String logClassName, Reporter reporter,
+      int notificationInterval, int timeout) {
+    this.logClassName = logClassName;
+    this.reporter = reporter;
+    this.notificationInterval = notificationInterval;
+    this.timeout = timeout;
   }
 
   public void go() {
     LOG.info("Running ReporterTask every " + notificationInterval
         + " miliseconds.");
     rpTimer = new Timer(true);
-    rpTimer.scheduleAtFixedRate(new ReporterTask(reporter), 0,
-        notificationInterval);
+
+
+    ReporterTask rt = new ReporterTask(reporter);
+    rpTimer.scheduleAtFixedRate(rt, 0, notificationInterval);
+
+    if (timeout > 0) {
+      srpTimer = new Timer(true);
+      StopReporterTimerTask srt = new StopReporterTimerTask(rt);
+      srpTimer.schedule(srt, timeout);
+    }
   }
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java Sat Sep
18 06:33:01 2010
@@ -309,7 +309,8 @@ public class ScriptOperator extends Oper
         if (HiveConf
             .getBoolVar(hconf, HiveConf.ConfVars.HIVESCRIPTAUTOPROGRESS)) {
           autoProgressor = new AutoProgressor(this.getClass().getName(),
-              reporter, Utilities.getDefaultNotificationInterval(hconf));
+              reporter, Utilities.getDefaultNotificationInterval(hconf),
+              HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) *
1000);
           autoProgressor.go();
         }
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java Sat Sep
18 06:33:01 2010
@@ -76,7 +76,8 @@ public class UDTFOperator extends Operat
     // for a while
     if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) {
       autoProgressor = new AutoProgressor(this.getClass().getName(), reporter,
-          Utilities.getDefaultNotificationInterval(hconf));
+          Utilities.getDefaultNotificationInterval(hconf),
+          HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) * 1000);
       autoProgressor.go();
     }
 



Mime
View raw message