hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1471674 - in /hive/branches/branch-0.11: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/profiler/ ql/src/test/org/apache/hadoop/hive/ql/exec/ ql/src/t...
Date Wed, 24 Apr 2013 21:02:16 GMT
Author: hashutosh
Date: Wed Apr 24 21:02:15 2013
New Revision: 1471674

URL: http://svn.apache.org/r1471674
Log:
HIVE-4318 : OperatorHooks hit performance even when not used (Gunther Hagleitner via Ashutosh
Chauhan)

Removed:
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
    hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java
    hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java
    hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
    hive/branches/branch-0.11/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/hiveprofiler0.q
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/hiveprofiler_script0.q
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/hiveprofiler_union0.q
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/operatorhook.q
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/hiveprofiler0.q.out
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/operatorhook.q.out
Modified:
    hive/branches/branch-0.11/build-common.xml
    hive/branches/branch-0.11/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/branches/branch-0.11/conf/hive-default.xml.template
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java

Modified: hive/branches/branch-0.11/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/build-common.xml?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/build-common.xml (original)
+++ hive/branches/branch-0.11/build-common.xml Wed Apr 24 21:02:15 2013
@@ -59,7 +59,7 @@
   <property name="test.output" value="true"/>
   <property name="test.junit.output.format" value="xml"/>
   <property name="test.junit.output.usefile" value="true"/>
-  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
+  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
   <property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q"
/>
   <property name="test.silent" value="true"/>
   <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>

Modified: hive/branches/branch-0.11/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/branch-0.11/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed
Apr 24 21:02:15 2013
@@ -190,7 +190,6 @@ public class HiveConf extends Configurat
     PREEXECHOOKS("hive.exec.pre.hooks", ""),
     POSTEXECHOOKS("hive.exec.post.hooks", ""),
     ONFAILUREHOOKS("hive.exec.failure.hooks", ""),
-    OPERATORHOOKS("hive.exec.operator.hooks", ""),
     CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""),
     EXECPARALLEL("hive.exec.parallel", false), // parallel query launching
     EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8),
@@ -565,18 +564,6 @@ public class HiveConf extends Configurat
     HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024
* 1024 * 1024), // 10G
     HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true),
 
-    //Profiler
-    HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"),
-    HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"),
-    HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring",
-        "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create
database
-    // default timeout for JDBC connection
-    HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30),
-    HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max",
-        0),     // maximum # of retries to insert/select/delete the stats DB
-    HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait",
-        3000),  // # milliseconds to wait before the next retry
-
     // Statistics
     HIVESTATSAUTOGATHER("hive.stats.autogather", true),
     HIVESTATSDBCLASS("hive.stats.dbclass",

Modified: hive/branches/branch-0.11/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/conf/hive-default.xml.template?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/conf/hive-default.xml.template (original)
+++ hive/branches/branch-0.11/conf/hive-default.xml.template Wed Apr 24 21:02:15 2013
@@ -1123,42 +1123,6 @@
 </property>
 
 <property>
-  <name>hive.profiler.dbclass</name>
-  <value>jdbc:derby</value>
-  <description>The default database that stores temporary hive statistics.</description>
-</property>
-
-<property>
-  <name>hive.profiler.jdbcdriver</name>
-  <value>org.apache.derby.jdbc.EmbeddedDriver</value>
-  <description>The JDBC driver for the database that stores temporary hive statistics.</description>
-</property>
-
-<property>
-  <name>hive.profiler.dbconnectionstring</name>
-  <value>jdbc:derby:;databaseName=TempStatsStore;create=true</value>
-  <description>The default connection string for the database that stores temporary
hive statistics.</description>
-</property>
-
-<property>
-  <name>hive.profiler.jdbc.timeout</name>
-  <value>30</value>
-  <description>Timeout value (number of seconds) used by JDBC connection and statements.</description>
-</property>
-
-<property>
-  <name>hive.profiler.retries.max</name>
-  <value>0</value>
-  <description>Maximum number of retries when profiler publisher/aggregator got an
exception updating intermediate database. Default is no tries on failures.</description>
-</property>
-
-<property>
-  <name>hive.profiler.retries.wait</name>
-  <value>3000</value>
-  <description>The base waiting window (in milliseconds) before the next retry. The
actual wait time is calculated by baseWindow * failues  baseWindow * (failure  1) * (random
number between [0.0,1.0]).</description>
-</property>
-
-<property>
   <name>hive.stats.dbclass</name>
   <value>jdbc:derby</value>
   <description>The default database that stores temporary hive statistics.</description>

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Wed
Apr 24 21:02:15 2013
@@ -50,7 +50,6 @@ public class ExecMapper extends MapReduc
   private JobConf jc;
   private boolean abort = false;
   private Reporter rp;
-  private List<OperatorHook> opHooks;
   public static final Log l4j = LogFactory.getLog("ExecMapper");
   private static boolean done;
 
@@ -99,7 +98,6 @@ public class ExecMapper extends MapReduc
       mo.setExecContext(execContext);
       mo.initializeLocalWork(jc);
       mo.initialize(jc, null);
-      opHooks = OperatorHookUtils.getOperatorHooks(jc);
 
       if (localWork == null) {
         return;
@@ -132,7 +130,6 @@ public class ExecMapper extends MapReduc
       rp = reporter;
       mo.setOutputCollector(oc);
       mo.setReporter(rp);
-      mo.setOperatorHooks(opHooks);
       MapredContext.get().setReporter(reporter);
     }
     // reset the execContext for each new row

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
Wed Apr 24 21:02:15 2013
@@ -66,7 +66,6 @@ public class ExecReducer extends MapRedu
   private long nextCntr = 1;
 
   private static String[] fieldNames;
-  private List<OperatorHook> opHooks;
   public static final Log l4j = LogFactory.getLog("ExecReducer");
   private boolean isLogInfoEnabled = false;
 
@@ -152,7 +151,6 @@ public class ExecReducer extends MapRedu
     try {
       l4j.info(reducer.dump(0));
       reducer.initialize(jc, rowObjectInspector);
-      opHooks = OperatorHookUtils.getOperatorHooks(jc);
     } catch (Throwable e) {
       abort = true;
       if (e instanceof OutOfMemoryError) {
@@ -183,7 +181,6 @@ public class ExecReducer extends MapRedu
       rp = reporter;
       reducer.setOutputCollector(oc);
       reducer.setReporter(rp);
-      reducer.setOperatorHooks(opHooks);
       MapredContext.get().setReporter(reporter);
     }
 

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1471674&r1=1471673&r2=1471674&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Wed
Apr 24 21:02:15 2013
@@ -55,7 +55,6 @@ public abstract class Operator<T extends
   // Bean methods
 
   private static final long serialVersionUID = 1L;
-  List<OperatorHook> operatorHooks;
 
   private Configuration configuration;
   protected List<Operator<? extends OperatorDesc>> childOperators;
@@ -241,17 +240,6 @@ public abstract class Operator<T extends
     return id;
   }
 
-  public void setOperatorHooks(List<OperatorHook> opHooks){
-    operatorHooks = opHooks;
-    if (childOperators == null) {
-      return;
-    }
-
-    for (Operator<? extends OperatorDesc> op : childOperators) {
-      op.setOperatorHooks(opHooks);
-    }
-  }
-
   public void setReporter(Reporter rep) {
     reporter = rep;
 
@@ -436,34 +424,6 @@ public abstract class Operator<T extends
     }
   }
 
-  private void enterOperatorHooks(OperatorHookContext opHookContext) throws HiveException
{
-    if (this.operatorHooks == null) {
-      return;
-    }
-    for(OperatorHook opHook : this.operatorHooks) {
-      opHook.enter(opHookContext);
-    }
-  }
-
-  private void exitOperatorHooks(OperatorHookContext opHookContext) throws HiveException
{
-    if (this.operatorHooks == null) {
-      return;
-    }
-    for(OperatorHook opHook : this.operatorHooks) {
-      opHook.exit(opHookContext);
-    }
-  }
-
-  private void closeOperatorHooks(OperatorHookContext opHookContext) throws HiveException
{
-    if (this.operatorHooks == null) {
-      return;
-    }
-    for(OperatorHook opHook : this.operatorHooks) {
-      opHook.close(opHookContext);
-    }
-  }
-
-
   /**
    * Collects all the parent's output object inspectors and calls actual
    * initialization method.
@@ -536,16 +496,10 @@ public abstract class Operator<T extends
       }
 
       beginTime = System.currentTimeMillis();
-      OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
-      enterOperatorHooks(opHookContext);
       processOp(row, tag);
-      exitOperatorHooks(opHookContext);
       totalTime += (System.currentTimeMillis() - beginTime);
     } else {
-      OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
-      enterOperatorHooks(opHookContext);
       processOp(row, tag);
-      exitOperatorHooks(opHookContext);
     }
   }
 
@@ -630,7 +584,6 @@ public abstract class Operator<T extends
 
     LOG.info(id + " forwarded " + cntr + " rows");
 
-    closeOperatorHooks(new OperatorHookContext(this));
     // call the operator specific close routine
     closeOp(abort);
 



Mime
View raw message