hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1471670 - in /hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec: FileSinkOperator.java GroupByOperator.java Operator.java
Date Wed, 24 Apr 2013 20:54:44 GMT
Author: hashutosh
Date: Wed Apr 24 20:54:44 2013
New Revision: 1471670

URL: http://svn.apache.org/r1471670
Log:
HIVE-4378 : Counters hit performance even when not used (Gunther Hagleitner via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1471670&r1=1471669&r2=1471670&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
Wed Apr 24 20:54:44 2013
@@ -754,7 +754,9 @@ public class FileSinkOperator extends Te
         // check # of dp
         if (valToPaths.size() > maxPartitions) {
           // throw fatal error
-          incrCounter(fatalErrorCntr, 1);
+          if (counterNameToEnum != null) {
+            incrCounter(fatalErrorCntr, 1);
+          }
           fatalError = true;
           LOG.error("Fatal error was thrown due to exceeding number of dynamic partitions");
         }

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=1471670&r1=1471669&r2=1471670&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
Wed Apr 24 20:54:44 2013
@@ -1076,7 +1076,7 @@ public class GroupByOperator extends Ope
       try {
         // put the hash related stats in statsMap if applicable, so that they
         // are sent to jt as counters
-        if (hashAggr) {
+        if (hashAggr && counterNameToEnum != null) {
           incrCounter(counterNameHashOut, numRowsHashTbl);
         }
 

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1471670&r1=1471669&r2=1471670&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Wed
Apr 24 20:54:44 2013
@@ -525,12 +525,28 @@ public abstract class Operator<T extends
     if (fatalError) {
       return;
     }
-    OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
-    preProcessCounter();
-    enterOperatorHooks(opHookContext);
-    processOp(row, tag);
-    exitOperatorHooks(opHookContext);
-    postProcessCounter();
+
+    if (counterNameToEnum != null) {
+      inputRows++;
+      if ((inputRows % 1000) == 0) {
+        incrCounter(numInputRowsCntr, inputRows);
+        incrCounter(timeTakenCntr, totalTime);
+        inputRows = 0;
+        totalTime = 0;
+      }
+
+      beginTime = System.currentTimeMillis();
+      OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
+      enterOperatorHooks(opHookContext);
+      processOp(row, tag);
+      exitOperatorHooks(opHookContext);
+      totalTime += (System.currentTimeMillis() - beginTime);
+    } else {
+      OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
+      enterOperatorHooks(opHookContext);
+      processOp(row, tag);
+      exitOperatorHooks(opHookContext);
+    }
   }
 
   // If a operator wants to do some work at the beginning of a group
@@ -606,9 +622,11 @@ public abstract class Operator<T extends
     state = State.CLOSE;
     LOG.info(id + " finished. closing... ");
 
-    incrCounter(numInputRowsCntr, inputRows);
-    incrCounter(numOutputRowsCntr, outputRows);
-    incrCounter(timeTakenCntr, totalTime);
+    if (counterNameToEnum != null) {
+      incrCounter(numInputRowsCntr, inputRows);
+      incrCounter(numOutputRowsCntr, outputRows);
+      incrCounter(timeTakenCntr, totalTime);
+    }
 
     LOG.info(id + " forwarded " + cntr + " rows");
 
@@ -822,9 +840,11 @@ public abstract class Operator<T extends
   protected void forward(Object row, ObjectInspector rowInspector)
       throws HiveException {
 
-    if ((++outputRows % 1000) == 0) {
-      incrCounter(numOutputRowsCntr, outputRows);
-      outputRows = 0;
+    if (counterNameToEnum != null) {
+      if ((++outputRows % 1000) == 0) {
+        incrCounter(numOutputRowsCntr, outputRows);
+        outputRows = 0;
+      }
     }
 
     if (isLogInfoEnabled) {
@@ -1158,39 +1178,12 @@ public abstract class Operator<T extends
   protected transient Object groupKeyObject;
 
   /**
-   * this is called before operator process to buffer some counters.
-   */
-  private void preProcessCounter() {
-    inputRows++;
-    if ((inputRows % 1000) == 0) {
-      incrCounter(numInputRowsCntr, inputRows);
-      incrCounter(timeTakenCntr, totalTime);
-      inputRows = 0;
-      totalTime = 0;
-    }
-    beginTime = System.currentTimeMillis();
-  }
-
-  /**
-   * this is called after operator process to buffer some counters.
-   */
-  private void postProcessCounter() {
-    if (counterNameToEnum != null) {
-      totalTime += (System.currentTimeMillis() - beginTime);
-    }
-  }
-
-  /**
    * this is called in operators in map or reduce tasks.
    *
    * @param name
    * @param amount
    */
   protected void incrCounter(String name, long amount) {
-    if(counterNameToEnum == null) {
-      return;
-    }
-
     String counterName = getWrappedCounterName(name);
     ProgressCounter pc = counterNameToEnum.get(counterName);
 



Mime
View raw message