Mailing-List: contact commits-help@hive.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hive-dev@hive.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1362782 - in /hive/trunk:
 common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
 conf/hive-default.xml.template
 ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
Date: Wed, 18 Jul 2012 06:36:19 -0000
To: commits@hive.apache.org
From: namit@apache.org
Message-Id: <20120718063619.B41262388860@eris.apache.org>

Author: namit
Date: Wed Jul 18 06:36:19 2012
New Revision: 1362782

URL: http://svn.apache.org/viewvc?rev=1362782&view=rev
Log:
HIVE-3230 Make logging of plan progress in HadoopJobExecHelper configurable
(Kevin Wilfong via namit)


Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml.template
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Jul 18 06:36:19 2012
@@ -387,6 +387,12 @@ public class HiveConf extends Configurat
     //Location of Hive run time structured log file
     HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")),
 
+    // Whether to log the plan's progress every time a job's progress is checked
+    HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", true),
+
+    // The interval between logging the plan's progress in milliseconds
+    HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", 60000L),
+
     // Default serde and record reader for user scripts
     HIVESCRIPTSERDE("hive.script.serde", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"),
     HIVESCRIPTRECORDREADER("hive.script.recordreader",

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Wed Jul 18 06:36:19 2012
@@ -708,6 +708,38 @@
 </property>
 
 <property>
+  <name>hive.querylog.location</name>
+  <value>/tmp/${user.name}</value>
+  <description>
+    Location of Hive run time structured log file
+  </description>
+</property>
+
+<property>
+  <name>hive.querylog.enable.plan.progress</name>
+  <value>true</value>
+  <description>
+    Whether to log the plan's progress every time a job's progress is checked.
+    These logs are written to the location specified by hive.querylog.location
+  </description>
+</property>
+
+<property>
+  <name>hive.querylog.plan.progress.interval</name>
+  <value>60000</value>
+  <description>
+    The interval to wait between logging the plan's progress in milliseconds.
+    If there is a whole number percentage change in the progress of the mappers or the reducers,
+    the progress is logged regardless of this value.
+    The actual interval will be the ceiling of (this value divided by the value of
+    hive.exec.counters.pull.interval) multiplied by the value of hive.exec.counters.pull.interval
+    I.e. if it is not divide evenly by the value of hive.exec.counters.pull.interval it will be
+    logged less frequently than specified.
+    This only has an effect if hive.querylog.enable.plan.progress is set to true.
+  </description>
+</property>
+
+<property>
   <name>hive.enforce.bucketing</name>
   <value>false</value>
   <description>Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced. </description>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java Wed Jul 18 06:36:19 2012
@@ -221,7 +221,8 @@ public class HadoopJobExecHelper {
     SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
     //DecimalFormat longFormatter = new DecimalFormat("###,###");
     long reportTime = System.currentTimeMillis();
-    long maxReportInterval = 60 * 1000; // One minute
+    long maxReportInterval =
+        HiveConf.getLongVar(job, HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL);
     boolean fatal = false;
     StringBuilder errMsg = new StringBuilder();
     long pullInterval = HiveConf.getLongVar(job, HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL);
@@ -355,8 +356,10 @@ public class HadoopJobExecHelper {
           ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
           ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
               Keys.TASK_HADOOP_PROGRESS, output);
-          ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
-          this.callBackObj.logPlanProgress(ss);
+          if (ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS)) {
+            ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
+            this.callBackObj.logPlanProgress(ss);
+          }
         }
         console.printInfo(output);
         lastReport = report;