hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1519797 - in /hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec: ./ vector/
Date Tue, 03 Sep 2013 19:03:43 GMT
Author: hashutosh
Date: Tue Sep  3 19:03:43 2013
New Revision: 1519797

URL: http://svn.apache.org/r1519797
Log:
HIVE-5152 : Vector operators should inherit from non-vector operators for code re-use. (Jitendra
Nath Pandey via Ashutosh Chauhan)

Modified:
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
Tue Sep  3 19:03:43 2013
@@ -46,13 +46,14 @@ public class FilterOperator extends Oper
     FILTERED, PASSED
   }
 
-  private final transient LongWritable filtered_count, passed_count;
+  protected final transient LongWritable filtered_count;
+  protected final transient LongWritable passed_count;
   private transient ExprNodeEvaluator conditionEvaluator;
   private transient PrimitiveObjectInspector conditionInspector;
   private transient int consecutiveFails;
   private transient int consecutiveSearches;
   private transient IOContext ioContext;
-  transient int heartbeatInterval;
+  protected transient int heartbeatInterval;
 
   public FilterOperator() {
     super();

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
Tue Sep  3 19:03:43 2013
@@ -142,8 +142,16 @@ public class GroupByOperator extends Ope
   transient StructObjectInspector newKeyObjectInspector;
   transient StructObjectInspector currentKeyObjectInspector;
   public static MemoryMXBean memoryMXBean;
-  private long maxMemory;
-  private float memoryThreshold;
+
+  /**
+   * Total amount of memory allowed for JVM heap.
+   */
+  protected long maxMemory;
+
+  /**
+   * configure percent of memory threshold usable by QP.
+   */
+  protected float memoryThreshold;
 
   private boolean groupingSetsPresent;
   private int groupingSetsPosition;
@@ -160,10 +168,18 @@ public class GroupByOperator extends Ope
   transient List<Field>[] aggrPositions;
 
   transient int fixedRowSize;
-  transient long maxHashTblMemory;
+
+  /**
+   * Max memory usable by the hashtable before it should flush.
+   */
+  protected transient long maxHashTblMemory;
   transient int totalVariableSize;
   transient int numEntriesVarSize;
-  transient int numEntriesHashTable;
+
+  /**
+   * Current number of entries in the hash table.
+   */
+  protected transient int numEntriesHashTable;
   transient int countAfterReport;
   transient int heartbeatInterval;
 

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
Tue Sep  3 19:03:43 2013
@@ -73,11 +73,11 @@ public class ReduceSinkOperator extends 
 
   // TODO: we use MetadataTypedColumnsetSerDe for now, till DynamicSerDe is
   // ready
-  transient Serializer keySerializer;
-  transient boolean keyIsText;
-  transient Serializer valueSerializer;
+  protected transient Serializer keySerializer;
+  protected transient boolean keyIsText;
+  protected transient Serializer valueSerializer;
   transient int tag;
-  transient byte[] tagByte = new byte[1];
+  protected transient byte[] tagByte = new byte[1];
   transient protected int numDistributionKeys;
   transient protected int numDistinctExprs;
   transient String inputAlias;  // input alias of this RS for join (used for PPD)
@@ -140,20 +140,20 @@ public class ReduceSinkOperator extends 
   }
 
   transient InspectableObject tempInspectableObject = new InspectableObject();
-  transient HiveKey keyWritable = new HiveKey();
-  transient Writable value;
+  protected transient HiveKey keyWritable = new HiveKey();
+  protected transient Writable value;
 
   transient StructObjectInspector keyObjectInspector;
   transient StructObjectInspector valueObjectInspector;
   transient ObjectInspector[] partitionObjectInspectors;
 
-  transient Object[][] cachedKeys;
-  transient Object[] cachedValues;
-  transient List<List<Integer>> distinctColIndices;
+  protected transient Object[][] cachedKeys;
+  protected transient Object[] cachedValues;
+  protected transient List<List<Integer>> distinctColIndices;
 
   boolean firstRow;
 
-  transient Random random;
+  protected transient Random random;
 
   /**
    * Initializes array of ExprNodeEvaluator. Adds Union field for distinct

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
Tue Sep  3 19:03:43 2013
@@ -18,11 +18,9 @@
 
 package org.apache.hadoop.hive.ql.exec.vector;
 
-import java.io.Serializable;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -30,27 +28,15 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.FilterDesc;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
-import org.apache.hadoop.io.LongWritable;
 
 /**
  * Filter operator implementation.
  **/
-public class VectorFilterOperator extends Operator<FilterDesc> implements
-    Serializable {
+public class VectorFilterOperator extends FilterOperator {
 
   private static final long serialVersionUID = 1L;
 
-  /**
-   * Counter.
-   *
-   */
-  public static enum Counter {
-    FILTERED, PASSED
-  }
-
-  private final transient LongWritable filtered_count, passed_count;
   private VectorExpression conditionEvaluator = null;
-  transient int heartbeatInterval;
 
   // filterMode is 1 if condition is always true, -1 if always false
   // and 0 if condition needs to be computed.
@@ -66,8 +52,6 @@ public class VectorFilterOperator extend
 
   public VectorFilterOperator() {
     super();
-    filtered_count = new LongWritable();
-    passed_count = new LongWritable();
     this.conf = (FilterDesc) conf;
   }
 
@@ -120,23 +104,10 @@ public class VectorFilterOperator extend
     }
   }
 
-  /**
-   * @return the name of the operator
-   */
-  @Override
-  public String getName() {
-    return getOperatorName();
-  }
-
   static public String getOperatorName() {
     return "FIL";
   }
 
-  @Override
-  public OperatorType getType() {
-    return OperatorType.FILTER;
-  }
-
   public VectorExpression getConditionEvaluator() {
     return conditionEvaluator;
   }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
Tue Sep  3 19:03:43 2013
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hive.ql.exec.vector;
 
-import java.io.Serializable;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.util.ArrayList;
@@ -31,8 +30,8 @@ import java.util.Map;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.KeyWrapper;
-import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
@@ -52,7 +51,7 @@ import org.apache.hadoop.hive.serde2.obj
  * stores the aggregate operators' intermediate states. Emits row mode output.
  *
  */
-public class VectorGroupByOperator extends Operator<GroupByDesc> implements Serializable
{
+public class VectorGroupByOperator extends GroupByOperator {
 
   private static final Log LOG = LogFactory.getLog(
       VectorGroupByOperator.class.getName());
@@ -83,21 +82,6 @@ public class VectorGroupByOperator exten
   private transient VectorHashKeyWrapperBatch keyWrappersBatch;
 
   /**
-   * Total amount of memory allowed for JVM heap.
-   */
-  private transient long maxMemory;
-
-  /**
-   * configure percent of memory threshold usable by QP.
-   */
-  private transient float memoryThreshold;
-
-  /**
-   * Max memory usable by the hashtable before it should flush.
-   */
-  private transient long maxHashTblMemory;
-
-  /**
    * Total per hashtable entry fixed memory (does not depend on key/agg values).
    */
   private transient int fixedHashEntrySize;
@@ -108,11 +92,6 @@ public class VectorGroupByOperator exten
   private transient int avgVariableSize;
 
   /**
-   * Current number of entries in the hash table.
-   */
-  private transient int numEntriesHashTable;
-
-  /**
    * Number of entries added to the hashtable since the last check if it should flush.
    */
   private transient int numEntriesSinceCheck;
@@ -434,23 +413,10 @@ public class VectorGroupByOperator exten
     }
   }
 
-  /**
-   * @return the name of the operator
-   */
-  @Override
-  public String getName() {
-    return getOperatorName();
-  }
-
   static public String getOperatorName() {
     return "GBY";
   }
 
-  @Override
-  public OperatorType getType() {
-    return OperatorType.GROUPBY;
-  }
-
   public VectorExpression[] getKeyExpressions() {
     return keyExpressions;
   }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
Tue Sep  3 19:03:43 2013
@@ -19,18 +19,15 @@
 package org.apache.hadoop.hive.ql.exec.vector;
 
 import java.io.IOException;
-import java.io.Serializable;
-import java.util.List;
 import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.TerminalOperator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
-import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
@@ -42,10 +39,8 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 
-public class VectorReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
-  implements Serializable {
+public class VectorReduceSinkOperator extends ReduceSinkOperator {
 
   private static final Log LOG = LogFactory.getLog(
       VectorReduceSinkOperator.class.getName());
@@ -89,28 +84,8 @@ public class VectorReduceSinkOperator ex
    */
   protected transient VectorExpressionWriter[] partitionWriters;
 
-  private transient int numDistributionKeys;
-
-  private transient List<List<Integer>> distinctColIndices;
-
-  private transient int numDistinctExprs;
-
-  transient HiveKey keyWritable = new HiveKey();
-  transient Writable value;
-
-  transient Object[] cachedValues;
-  transient Object[][] cachedKeys;
-  transient Random random;
-
-  transient Serializer keySerializer;
-  transient boolean keyIsText;
-  transient Serializer valueSerializer;
-  transient int tag;
-  transient byte[] tagByte = new byte[1];
-
   transient ObjectInspector keyObjectInspector;
   transient ObjectInspector valueObjectInspector;
-  transient ObjectInspector[] partitionObjectInspectors;
   transient int [] keyHashCode = new int [VectorizedRowBatch.DEFAULT_SIZE];
 
   public VectorReduceSinkOperator(VectorizationContext vContext, OperatorDesc conf)
@@ -331,28 +306,10 @@ public class VectorReduceSinkOperator ex
     }
   }
 
-  /**
-   * @return the name of the operator
-   */
-  @Override
-  public String getName() {
-    return getOperatorName();
-  }
-
   static public String getOperatorName() {
     return "RS";
   }
 
-  @Override
-  public OperatorType getType() {
-    return OperatorType.REDUCESINK;
-  }
-
-  @Override
-  public boolean opAllowedBeforeMapJoin() {
-    return false;
-  }
-
   public VectorExpression[] getPartitionEval() {
     return partitionEval;
   }
@@ -376,5 +333,4 @@ public class VectorReduceSinkOperator ex
   public void setKeyEval(VectorExpression[] keyEval) {
     this.keyEval = keyEval;
   }
-
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1519797&r1=1519796&r2=1519797&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
(original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
Tue Sep  3 19:03:43 2013
@@ -18,12 +18,11 @@
 
 package org.apache.hadoop.hive.ql.exec.vector;
 
-import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
@@ -39,8 +38,7 @@ import org.apache.hadoop.hive.serde2.obj
 /**
  * Select operator implementation.
  */
-public class VectorSelectOperator extends Operator<SelectDesc> implements
-    Serializable {
+public class VectorSelectOperator extends SelectOperator {
 
   private static final long serialVersionUID = 1L;
 
@@ -128,23 +126,10 @@ public class VectorSelectOperator extend
     vrg.valueWriters = originalValueWriters;
   }
 
-  /**
-   * @return the name of the operator
-   */
-  @Override
-  public String getName() {
-    return getOperatorName();
-  }
-
   static public String getOperatorName() {
     return "SEL";
   }
 
-  @Override
-  public OperatorType getType() {
-    return OperatorType.SELECT;
-  }
-
   @Explain (displayName = "vector expressions")
   public VectorExpression[] getvExpressions() {
     return vExpressions;



Mime
View raw message