hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r904321 - in /hadoop/hive/trunk: CHANGES.txt ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
Date Fri, 29 Jan 2010 00:29:23 GMT
Author: namit
Date: Fri Jan 29 00:29:22 2010
New Revision: 904321

URL: http://svn.apache.org/viewvc?rev=904321&view=rev
Log:
HIVE-1110. Add a counter to denote that skewjoin got triggered
(He Yongqiang via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=904321&r1=904320&r2=904321&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jan 29 00:29:22 2010
@@ -95,6 +95,9 @@
     input size of skew join's following map join job
     (He Yongqiang via namit)
 
+    HIVE-1110. Add a counter to denote that skewjoin got triggered
+    (He Yongqiang via namit)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=904321&r1=904320&r2=904321&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Fri Jan
29 00:29:22 2010
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.LongWritable;
 
 /**
  * Join operator implementation.
@@ -41,7 +42,12 @@
   private static final long serialVersionUID = 1L;
 
   private transient SkewJoinHandler skewJoinKeyContext = null;
-
+  
+  public static enum SkewkeyTableCounter {
+    SKEWJOINFOLLOWUPJOBS
+  }
+  transient private final LongWritable skewjoin_followup_jobs = new LongWritable(0);
+  
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
     super.initializeOp(hconf);
@@ -49,7 +55,9 @@
     if (handleSkewJoin) {
       skewJoinKeyContext = new SkewJoinHandler(this);
       skewJoinKeyContext.initiliaze(hconf);
+      skewJoinKeyContext.setSkewJoinJobCounter(skewjoin_followup_jobs);
     }
+    statsMap.put(SkewkeyTableCounter.SKEWJOINFOLLOWUPJOBS, skewjoin_followup_jobs);
   }
 
   @Override

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java?rev=904321&r1=904320&r2=904321&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java Fri
Jan 29 00:29:22 2010
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.util.ReflectionUtils;
 
 /**
@@ -85,6 +86,8 @@
   private Map<Byte, TableDesc> tblDesc = null;
 
   private Map<Byte, Boolean> bigKeysExistingMap = null;
+  
+  private LongWritable skewjoinFollowupJobs;
 
   Configuration hconf = null;
   List<Object> dummyKey = null;
@@ -226,7 +229,7 @@
       // table (the last table can always be streamed), we define that we get
       // a skew key now.
       currBigKeyTag = tag;
-
+      updateSkewJoinJobCounter(tag);
       // right now we assume that the group by is an ArrayList object. It may
       // change in future.
       if (!(dummyKey instanceof List)) {
@@ -342,4 +345,12 @@
     return new Path(tmpPath, taskId);
   }
 
+  public void setSkewJoinJobCounter(LongWritable skewjoinFollowupJobs) {
+    this.skewjoinFollowupJobs = skewjoinFollowupJobs;
+  }
+  
+  public void updateSkewJoinJobCounter(int tag) {
+    this.skewjoinFollowupJobs.set(this.skewjoinFollowupJobs.get()+1);
+  }
+
 }



Mime
View raw message