hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1581482 - in /hive/branches/branch-0.13: common/src/java/org/apache/hadoop/hive/common/type/ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ ql/src/java/org/apache/hadoop/hi...
Date Tue, 25 Mar 2014 19:40:18 GMT
Author: jitendra
Date: Tue Mar 25 19:40:17 2014
New Revision: 1581482

URL: http://svn.apache.org/r1581482
Log:
HIVE-6349 : Column name map is broken (reviewed by Remus, Sergey)

Added:
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out
Modified:
    hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearLong.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
(original)
+++ hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
Tue Mar 25 19:40:17 2014
@@ -273,13 +273,4 @@ public class HiveDecimal implements Comp
 
     return bd;
   }
-
-  /**
-   * Sets the {@link BigDecimal} value in this object.
-   * @param bigDecimal
-   */
-  public void setNormalize(BigDecimal bigDecimal) {
-    BigDecimal value = normalize(bigDecimal, true);
-    this.bd = value;
-  }
 }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
Tue Mar 25 19:40:17 2014
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.serde2.obj
 /**
  * Select operator implementation.
  */
-public class VectorSelectOperator extends SelectOperator {
+public class VectorSelectOperator extends SelectOperator implements VectorizationContextRegion
{
 
   private static final long serialVersionUID = 1L;
 
@@ -48,6 +48,9 @@ public class VectorSelectOperator extend
 
   private transient VectorExpressionWriter [] valueWriters = null;
 
+  // Create a new outgoing vectorization context because column name map will change.
+  private VectorizationContext vOutContext;
+
   public VectorSelectOperator(VectorizationContext vContext, OperatorDesc conf)
       throws HiveException {
     this.conf = (SelectDesc) conf;
@@ -59,14 +62,21 @@ public class VectorSelectOperator extend
       vExpressions[i] = ve;
     }
 
-    Map<String, Integer> cMap = vContext.getColumnMap();
+    /**
+     * Create a new vectorization context to update the column map but same output column
manager
+     * must be inherited to track the scratch the columns.
+     */
+    vOutContext = new VectorizationContext(vContext);
+
+    // Set a fileKey, although this operator doesn't use it.
+    vOutContext.setFileKey(vContext.getFileKey() + "/_SELECT_");
+
+    // Update column map
+    vOutContext.getColumnMap().clear();
     for (int i=0; i < colList.size(); ++i) {
       String columnName = this.conf.getOutputColumnNames().get(i);
-      if (!cMap.containsKey(columnName)) {
-        VectorExpression ve = vExpressions[i];
-        // Update column map with output column names
-        vContext.addToColumnMap(columnName, ve.getOutputColumn());
-      }
+      VectorExpression ve = vExpressions[i];
+      vOutContext.addToColumnMap(columnName, ve.getOutputColumn());
     }
   }
 
@@ -153,4 +163,9 @@ public class VectorSelectOperator extend
   public void setVExpressions(VectorExpression[] vExpressions) {
     this.vExpressions = vExpressions;
   }
+
+  @Override
+  public VectorizationContext getOuputVectorizationContext() {
+    return vOutContext;
+  }
 }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
Tue Mar 25 19:40:17 2014
@@ -120,6 +120,11 @@ public class VectorizationContext {
   //Map column number to type
   private final OutputColumnManager ocm;
 
+  // File key is used by operators to retrieve the scratch vectors
+  // from mapWork at runtime. The operators that modify the structure of
+  // a vector row batch, need to allocate scratch vectors as well. Every
+  // operator that creates a new Vectorization context should set a unique
+  // fileKey.
   private String fileKey = null;
 
   // Set of UDF classes for type casting data types in row-mode.
@@ -151,6 +156,19 @@ public class VectorizationContext {
     vMap = new VectorExpressionDescriptor();
   }
 
+  /**
+   * This constructor inherits the OutputColumnManger and from
+   * the 'parent' constructor, therefore this should be used only by operators
+   * that don't create a new vectorized row batch. This should be used only by
+   * operators that want to modify the columnName map without changing the row batch.
+   */
+  public VectorizationContext(VectorizationContext parent) {
+    this.columnMap = new HashMap<String, Integer>(parent.columnMap);
+    this.ocm = parent.ocm;
+    this.firstOutputColumnIndex = parent.firstOutputColumnIndex;
+    vMap = new VectorExpressionDescriptor();
+  }
+
   public String getFileKey() {
     return fileKey;
   }
@@ -170,7 +188,7 @@ public class VectorizationContext {
     return columnMap.get(colExpr.getColumn());
   }
 
-  private class OutputColumnManager {
+  private static class OutputColumnManager {
     private final int initialOutputCol;
     private int outputColCount = 0;
 

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
Tue Mar 25 19:40:17 2014
@@ -447,28 +447,24 @@ public final class VectorExpressionWrite
       SettableHiveDecimalObjectInspector fieldObjInspector) throws HiveException {
 
     return new VectorExpressionWriterDecimal() {
-      private HiveDecimal hd;
       private Object obj;
 
       public VectorExpressionWriter init(SettableHiveDecimalObjectInspector objInspector)
throws HiveException {
         super.init(objInspector);
-        hd = HiveDecimal.create(BigDecimal.ZERO);
         obj = initValue(null);
         return this;
       }
 
       @Override
       public Object writeValue(Decimal128 value) throws HiveException {
-        hd.setNormalize(value.toBigDecimal());
-        ((SettableHiveDecimalObjectInspector) this.objectInspector).set(obj, hd);
-        return obj;
+        return ((SettableHiveDecimalObjectInspector) this.objectInspector).set(obj,
+            HiveDecimal.create(value.toBigDecimal()));
       }
 
       @Override
       public Object setValue(Object field, Decimal128 value) {
-        hd.setNormalize(value.toBigDecimal());
-        ((SettableHiveDecimalObjectInspector) this.objectInspector).set(field, hd);
-        return field;
+        return ((SettableHiveDecimalObjectInspector) this.objectInspector).set(field,
+            HiveDecimal.create(value.toBigDecimal()));
       }
 
       @Override

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearLong.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearLong.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearLong.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearLong.java
Tue Mar 25 19:40:17 2014
@@ -29,16 +29,16 @@ public final class VectorUDFYearLong ext
 
   private static final long serialVersionUID = 1L;
   /* year boundaries in nanoseconds */
-  static transient final long[] YEAR_BOUNDARIES;
-  static transient final int MIN_YEAR = 1901;
-  static transient final int MAX_YEAR = 2038;
+  private static transient final long[] YEAR_BOUNDARIES;
+  private static transient final int MIN_YEAR = 1678;
+  private static transient final int MAX_YEAR = 2300;
 
   static {
     YEAR_BOUNDARIES = new long[MAX_YEAR-MIN_YEAR];
     Calendar c = Calendar.getInstance();
     c.setTimeInMillis(0); // c.set doesn't reset millis
     /* 1901 Jan is not with in range */
-    for(int year=MIN_YEAR+1; year <= 2038; year++) {
+    for(int year=MIN_YEAR+1; year <= MAX_YEAR; year++) {
       c.set(year, Calendar.JANUARY, 1, 0, 0, 0);
       YEAR_BOUNDARIES[year-MIN_YEAR-1] = c.getTimeInMillis()*1000*1000;
     }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1581482&r1=1581481&r2=1581482&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
Tue Mar 25 19:40:17 2014
@@ -33,7 +33,6 @@ import java.util.regex.Pattern;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.exec.*;
 import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
 import org.apache.hadoop.hive.ql.exec.tez.TezTask;
@@ -53,11 +52,7 @@ import org.apache.hadoop.hive.ql.lib.Rul
 import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc;
 import org.apache.hadoop.hive.ql.plan.AggregationDesc;
@@ -378,7 +373,9 @@ public class Vectorizer implements Physi
   class VectorizationNodeProcessor implements NodeProcessor {
 
     private final MapWork mWork;
-    private final Map<String, VectorizationContext> vectorizationContexts =
+
+    // This is used to extract scratch column types for each file key
+    private final Map<String, VectorizationContext> scratchColumnContext =
         new HashMap<String, VectorizationContext>();
 
     private final Map<Operator<? extends OperatorDesc>, VectorizationContext>
vContextsByTSOp =
@@ -394,8 +391,8 @@ public class Vectorizer implements Physi
     public Map<String, Map<Integer, String>> getScratchColumnVectorTypes() {
       Map<String, Map<Integer, String>> scratchColumnVectorTypes =
           new HashMap<String, Map<Integer, String>>();
-      for (String onefile : vectorizationContexts.keySet()) {
-        VectorizationContext vc = vectorizationContexts.get(onefile);
+      for (String onefile : scratchColumnContext.keySet()) {
+        VectorizationContext vc = scratchColumnContext.get(onefile);
         Map<Integer, String> cmap = vc.getOutputColumnTypeMap();
         scratchColumnVectorTypes.put(onefile, cmap);
       }
@@ -405,8 +402,8 @@ public class Vectorizer implements Physi
     public Map<String, Map<String, Integer>> getScratchColumnMap() {
       Map<String, Map<String, Integer>> scratchColumnMap =
           new HashMap<String, Map<String, Integer>>();
-      for(String oneFile: vectorizationContexts.keySet()) {
-        VectorizationContext vc = vectorizationContexts.get(oneFile);
+      for(String oneFile: scratchColumnContext.keySet()) {
+        VectorizationContext vc = scratchColumnContext.get(oneFile);
         Map<String, Integer> cmap = vc.getColumnMap();
         scratchColumnMap.put(oneFile, cmap);
       }
@@ -433,7 +430,7 @@ public class Vectorizer implements Physi
               // Each partition gets a copy
               //
               vContext.setFileKey(onefile);
-              vectorizationContexts.put(onefile, vContext);
+              scratchColumnContext.put(onefile, vContext);
               break;
             }
           }
@@ -472,7 +469,7 @@ public class Vectorizer implements Physi
               VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
               VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
               vContextsByTSOp.put(op, vOutContext);
-              vectorizationContexts.put(vOutContext.getFileKey(), vOutContext);
+              scratchColumnContext.put(vOutContext.getFileKey(), vOutContext);
             }
           }
         } catch (HiveException e) {

Added: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q?rev=1581482&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q
(added)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q
Tue Mar 25 19:40:17 2014
@@ -0,0 +1,8 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.nonconditionaltask=true;
+SET hive.auto.convert.join.nonconditionaltask.size=1000000000;
+
+explain select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble as td from alltypesorc
v1, alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc v3 on t1.tsi=v3.csmallint;
+
+select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble as td from alltypesorc v1,
alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc v3 on t1.tsi=v3.csmallint;

Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out?rev=1581482&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out
(added)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out
Tue Mar 25 19:40:17 2014
@@ -0,0 +1,125 @@
+PREHOOK: query: explain select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble as
td from alltypesorc v1, alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc
v3 on t1.tsi=v3.csmallint
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble
as td from alltypesorc v1, alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc
v3 on t1.tsi=v3.csmallint
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-7 is a root stage
+  Stage-2 depends on stages: Stage-7
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-7
+    Map Reduce Local Work
+      Alias -> Map Local Tables:
+        t1:v1 
+          Fetch Operator
+            limit: -1
+        v3 
+          Fetch Operator
+            limit: -1
+      Alias -> Map Local Operator Tree:
+        t1:v1 
+          TableScan
+            alias: v1
+            Statistics: Num rows: 23577 Data size: 377237 Basic stats: COMPLETE Column stats:
NONE
+            HashTable Sink Operator
+              condition expressions:
+                0 {ctinyint} {csmallint} {cdouble}
+                1 {ctinyint}
+              keys:
+                0 ctinyint (type: tinyint)
+                1 ctinyint (type: tinyint)
+        v3 
+          TableScan
+            alias: v3
+            Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats:
NONE
+            HashTable Sink Operator
+              condition expressions:
+                0 {_col1}
+                1 
+              keys:
+                0 _col0 (type: smallint)
+                1 csmallint (type: smallint)
+
+  Stage: Stage-2
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: v2
+            Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats:
NONE
+            Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              condition expressions:
+                0 {ctinyint} {csmallint} {cdouble}
+                1 {ctinyint}
+              keys:
+                0 ctinyint (type: tinyint)
+                1 ctinyint (type: tinyint)
+              outputColumnNames: _col0, _col1, _col5, _col14
+              Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column
stats: NONE
+              Filter Operator
+                predicate: (_col0 = _col14) (type: boolean)
+                Statistics: Num rows: 51869 Data size: 207477 Basic stats: COMPLETE Column
stats: NONE
+                Select Operator
+                  expressions: _col1 (type: smallint), _col5 (type: double)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 51869 Data size: 207477 Basic stats: COMPLETE Column
stats: NONE
+                  Map Join Operator
+                    condition map:
+                         Inner Join 0 to 1
+                    condition expressions:
+                      0 {_col1}
+                      1 
+                    keys:
+                      0 _col0 (type: smallint)
+                      1 csmallint (type: smallint)
+                    outputColumnNames: _col1
+                    Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: _col1 (type: double)
+                      outputColumnNames: _col1
+                      Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE
Column stats: NONE
+                      Group By Operator
+                        aggregations: sum(_col1)
+                        mode: hash
+                        outputColumnNames: _col0
+                        Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+                        Reduce Output Operator
+                          sort order: 
+                          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+                          value expressions: _col0 (type: double)
+      Local Work:
+        Map Reduce Local Work
+      Execution mode: vectorized
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: sum(VALUE._col0)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: _col0 (type: double)
+            outputColumnNames: _col0
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+PREHOOK: query: select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble as td from
alltypesorc v1, alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc v3 on t1.tsi=v3.csmallint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(t1.td) from (select  v1.csmallint as tsi, v1.cdouble as td from
alltypesorc v1, alltypesorc v2 where v1.ctinyint=v2.ctinyint) t1 join alltypesorc v3 on t1.tsi=v3.csmallint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+6.065190932486892E11



Mime
View raw message