hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1673337 [3/10] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/org/apache/hive/common/util/ itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/...
Date Tue, 14 Apr 2015 01:59:31 GMT
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Apr 14 01:59:29 2015
@@ -136,7 +136,8 @@ public class Vectorizer implements Physi
 
   Set<String> supportedAggregationUdfs = new HashSet<String>();
 
-  private PhysicalContext physicalContext = null;;
+  private PhysicalContext physicalContext = null;
+  private HiveConf hiveConf;
 
   public Vectorizer() {
 
@@ -286,13 +287,13 @@ public class Vectorizer implements Physi
 
   class VectorizationDispatcher implements Dispatcher {
 
-    private final PhysicalContext pctx;
+    private final PhysicalContext physicalContext;
 
     private List<String> reduceColumnNames;
     private List<TypeInfo> reduceTypeInfos;
 
-    public VectorizationDispatcher(PhysicalContext pctx) {
-      this.pctx = pctx;
+    public VectorizationDispatcher(PhysicalContext physicalContext) {
+      this.physicalContext = physicalContext;
       reduceColumnNames = null;
       reduceTypeInfos = null;
     }
@@ -310,7 +311,7 @@ public class Vectorizer implements Physi
             convertMapWork((MapWork) w, true);
           } else if (w instanceof ReduceWork) {
             // We are only vectorizing Reduce under Tez.
-            if (HiveConf.getBoolVar(pctx.getConf(),
+            if (HiveConf.getBoolVar(hiveConf,
                         HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED)) {
               convertReduceWork((ReduceWork) w);
             }
@@ -322,7 +323,7 @@ public class Vectorizer implements Physi
           if (baseWork instanceof MapWork) {
             convertMapWork((MapWork) baseWork, false);
           } else if (baseWork instanceof ReduceWork
-              && HiveConf.getBoolVar(pctx.getConf(),
+              && HiveConf.getBoolVar(hiveConf,
                   HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED)) {
             convertReduceWork((ReduceWork) baseWork);
           }
@@ -393,13 +394,12 @@ public class Vectorizer implements Physi
       HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
       ogw.startWalking(topNodes, nodeOutput);
 
-      Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps();
-      mapWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps);
-      Map<String, Map<String, Integer>> allColumnVectorMaps = vnp.getAllColumnVectorMaps();
-      mapWork.setAllColumnVectorMaps(allColumnVectorMaps);
+      mapWork.setVectorColumnNameMap(vnp.getVectorColumnNameMap());
+      mapWork.setVectorColumnTypeMap(vnp.getVectorColumnTypeMap());
+      mapWork.setVectorScratchColumnTypeMap(vnp.getVectorScratchColumnTypeMap());
 
       if (LOG.isDebugEnabled()) {
-        debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps);
+        debugDisplayAllMaps(mapWork);
       }
 
       return;
@@ -495,7 +495,7 @@ public class Vectorizer implements Physi
       // VectorizationContext...  Do we use PreOrderWalker instead of DefaultGraphWalker.
       Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
       ReduceWorkVectorizationNodeProcessor vnp =
-              new ReduceWorkVectorizationNodeProcessor(reduceColumnNames);
+              new ReduceWorkVectorizationNodeProcessor(reduceColumnNames, reduceTypeInfos);
       addReduceWorkRules(opRules, vnp);
       Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
       GraphWalker ogw = new PreOrderWalker(disp);
@@ -510,14 +510,12 @@ public class Vectorizer implements Physi
       // Necessary since we are vectorizing the root operator in reduce.
       reduceWork.setReducer(vnp.getRootVectorOp());
 
-      Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps();
-      reduceWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps);
-      Map<String, Map<String, Integer>> allColumnVectorMaps = vnp.getAllColumnVectorMaps();
-      reduceWork.setAllColumnVectorMaps(allColumnVectorMaps);
-
+      reduceWork.setVectorColumnNameMap(vnp.getVectorColumnNameMap());
+      reduceWork.setVectorColumnTypeMap(vnp.getVectorColumnTypeMap());
+      reduceWork.setVectorScratchColumnTypeMap(vnp.getVectorScratchColumnTypeMap());
 
       if (LOG.isDebugEnabled()) {
-        debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps);
+        debugDisplayAllMaps(reduceWork);
       }
     }
   }
@@ -574,37 +572,33 @@ public class Vectorizer implements Physi
   // ReduceWorkVectorizationNodeProcessor.
   class VectorizationNodeProcessor implements NodeProcessor {
 
-    // This is used to extract scratch column types for each file key
-    protected final Map<String, VectorizationContext> scratchColumnContext =
-        new HashMap<String, VectorizationContext>();
+    // The vectorization context for the Map or Reduce task.
+    protected VectorizationContext taskVectorizationContext;
+
+    // The input projection column type name map for the Map or Reduce task.
+    protected Map<Integer, String> taskColumnTypeNameMap;
+
+    VectorizationNodeProcessor() {
+      taskColumnTypeNameMap = new HashMap<Integer, String>();
+    }
 
-    protected final Map<Operator<? extends OperatorDesc>, VectorizationContext> vContextsByOp =
-        new HashMap<Operator<? extends OperatorDesc>, VectorizationContext>();
+    public Map<String, Integer> getVectorColumnNameMap() {
+      return taskVectorizationContext.getProjectionColumnMap();
+    }
+
+    public Map<Integer, String> getVectorColumnTypeMap() {
+      return taskColumnTypeNameMap;
+    }
+
+    public Map<Integer, String> getVectorScratchColumnTypeMap() {
+      return taskVectorizationContext.getScratchColumnTypeMap();
+    }
 
     protected final Set<Operator<? extends OperatorDesc>> opsDone =
         new HashSet<Operator<? extends OperatorDesc>>();
 
-    public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
-      Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps =
-          new HashMap<String, Map<Integer, String>>();
-      for (String onefile : scratchColumnContext.keySet()) {
-        VectorizationContext vc = scratchColumnContext.get(onefile);
-        Map<Integer, String> cmap = vc.getScratchColumnTypeMap();
-        allScratchColumnVectorTypeMaps.put(onefile, cmap);
-      }
-      return allScratchColumnVectorTypeMaps;
-    }
-
-    public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
-      Map<String, Map<String, Integer>> allColumnVectorMaps =
-          new HashMap<String, Map<String, Integer>>();
-      for(String oneFile: scratchColumnContext.keySet()) {
-        VectorizationContext vc = scratchColumnContext.get(oneFile);
-        Map<String, Integer> cmap = vc.getProjectionColumnMap();
-        allColumnVectorMaps.put(oneFile, cmap);
-      }
-      return allColumnVectorMaps;
-    }
+    protected final Map<Operator<? extends OperatorDesc>, Operator<? extends OperatorDesc>> opToVectorOpMap =
+        new HashMap<Operator<? extends OperatorDesc>, Operator<? extends OperatorDesc>>();
 
     public VectorizationContext walkStackToFindVectorizationContext(Stack<Node> stack,
             Operator<? extends OperatorDesc> op) throws SemanticException {
@@ -622,7 +616,18 @@ public class Vectorizer implements Physi
           return null;
         }
         Operator<? extends OperatorDesc> opParent = (Operator<? extends OperatorDesc>) stack.get(i);
-        vContext = vContextsByOp.get(opParent);
+        Operator<? extends OperatorDesc> vectorOpParent = opToVectorOpMap.get(opParent);
+        if (vectorOpParent != null) {
+          if (vectorOpParent instanceof VectorizationContextRegion) {
+            VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOpParent;
+            vContext = vcRegion.getOuputVectorizationContext();
+            LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " has new vectorization context " + vContext.toString());
+          } else {
+            LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " does not have new vectorization context");
+          }
+        } else {
+          LOG.info("walkStackToFindVectorizationContext " + opParent.getName() + " is not vectorized");
+        }
         --i;
       }
       return vContext;
@@ -636,14 +641,9 @@ public class Vectorizer implements Physi
           vectorOp = vectorizeOperator(op, vContext);
           opsDone.add(op);
           if (vectorOp != op) {
+            opToVectorOpMap.put(op, vectorOp);
             opsDone.add(vectorOp);
           }
-          if (vectorOp instanceof VectorizationContextRegion) {
-            VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
-            VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
-            vContextsByOp.put(op, vOutContext);
-            scratchColumnContext.put(vOutContext.getFileKey(), vOutContext);
-          }
         }
       } catch (HiveException e) {
         throw new SemanticException(e);
@@ -663,6 +663,7 @@ public class Vectorizer implements Physi
     private final MapWork mWork;
 
     public MapWorkVectorizationNodeProcessor(MapWork mWork) {
+      super();
       this.mWork = mWork;
     }
 
@@ -671,41 +672,26 @@ public class Vectorizer implements Physi
         Object... nodeOutputs) throws SemanticException {
 
       Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
-      LOG.info("MapWorkVectorizationNodeProcessor processing Operator: " + op.getName() + "...");
 
       VectorizationContext vContext = null;
 
       if (op instanceof TableScanOperator) {
-        vContext = getVectorizationContext(op, physicalContext);
-        for (String onefile : mWork.getPathToAliases().keySet()) {
-          List<String> aliases = mWork.getPathToAliases().get(onefile);
-          for (String alias : aliases) {
-            Operator<? extends OperatorDesc> opRoot = mWork.getAliasToWork().get(alias);
-            if (op == opRoot) {
-              // The same vectorization context is copied multiple times into
-              // the MapWork scratch columnMap
-              // Each partition gets a copy
-              //
-              vContext.setFileKey(onefile);
-              scratchColumnContext.put(onefile, vContext);
-              if (LOG.isDebugEnabled()) {
-                LOG.debug("Vectorized MapWork operator " + op.getName() + " vectorization context " + vContext.toString());
-              }
-              break;
-            }
-          }
+        if (taskVectorizationContext == null) {
+          taskVectorizationContext = getVectorizationContext(op.getSchema(), op.getName(),
+                  taskColumnTypeNameMap);
         }
-        vContextsByOp.put(op, vContext);
+        vContext = taskVectorizationContext;
       } else {
+        LOG.info("MapWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName());
         vContext = walkStackToFindVectorizationContext(stack, op);
         if (vContext == null) {
-          throw new SemanticException(
-              String.format("Did not find vectorization context for operator %s in operator stack",
-                      op.getName()));
+          // No operator has "pushed" a new context -- so use the task vectorization context.
+          vContext = taskVectorizationContext;
         }
       }
 
       assert vContext != null;
+      LOG.info("MapWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString());
 
       // When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
       // vectorize the operators below it.
@@ -720,9 +706,10 @@ public class Vectorizer implements Physi
       Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
 
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
         if (vectorOp instanceof VectorizationContextRegion) {
-          LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
+          VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+          VectorizationContext vNewContext = vcRegion.getOuputVectorizationContext();
+          LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vNewContext.toString());
         }
       }
 
@@ -733,8 +720,7 @@ public class Vectorizer implements Physi
   class ReduceWorkVectorizationNodeProcessor extends VectorizationNodeProcessor {
 
     private final List<String> reduceColumnNames;
-
-    private VectorizationContext reduceShuffleVectorizationContext;
+    private final List<TypeInfo> reduceTypeInfos;
 
     private Operator<? extends OperatorDesc> rootVectorOp;
 
@@ -742,10 +728,12 @@ public class Vectorizer implements Physi
       return rootVectorOp;
     }
 
-    public ReduceWorkVectorizationNodeProcessor(List<String> reduceColumnNames) {
+    public ReduceWorkVectorizationNodeProcessor(List<String> reduceColumnNames,
+            List<TypeInfo> reduceTypeInfos) {
+      super();
       this.reduceColumnNames =  reduceColumnNames;
+      this.reduceTypeInfos = reduceTypeInfos;
       rootVectorOp = null;
-      reduceShuffleVectorizationContext = null;
     }
 
     @Override
@@ -753,8 +741,6 @@ public class Vectorizer implements Physi
         Object... nodeOutputs) throws SemanticException {
 
       Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
-      LOG.info("ReduceWorkVectorizationNodeProcessor processing Operator: " +
-              op.getName() + "...");
 
       VectorizationContext vContext = null;
 
@@ -763,25 +749,30 @@ public class Vectorizer implements Physi
       if (op.getParentOperators().size() == 0) {
         LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + reduceColumnNames.toString());
 
-        vContext = new VectorizationContext(reduceColumnNames);
-        vContext.setFileKey("_REDUCE_SHUFFLE_");
-        scratchColumnContext.put("_REDUCE_SHUFFLE_", vContext);
-        reduceShuffleVectorizationContext = vContext;
+        vContext = new VectorizationContext("__Reduce_Shuffle__", reduceColumnNames);
+        taskVectorizationContext = vContext;
+        int i = 0;
+        for (TypeInfo typeInfo : reduceTypeInfos) {
+          taskColumnTypeNameMap.put(i, typeInfo.getTypeName());
+          i++;
+        }
         saveRootVectorOp = true;
 
         if (LOG.isDebugEnabled()) {
           LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context " + vContext.toString());
         }
       } else {
+        LOG.info("ReduceWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName());
         vContext = walkStackToFindVectorizationContext(stack, op);
         if (vContext == null) {
           // If we didn't find a context among the operators, assume the top -- reduce shuffle's
           // vectorization context.
-          vContext = reduceShuffleVectorizationContext;
+          vContext = taskVectorizationContext;
         }
       }
 
       assert vContext != null;
+      LOG.info("ReduceWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString());
 
       // When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
       // vectorize the operators below it.
@@ -796,9 +787,10 @@ public class Vectorizer implements Physi
       Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
 
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
         if (vectorOp instanceof VectorizationContextRegion) {
-          LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
+          VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+          VectorizationContext vNewContext = vcRegion.getOuputVectorizationContext();
+          LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vNewContext.toString());
         }
       }
       if (vectorOp instanceof VectorGroupByOperator) {
@@ -816,7 +808,7 @@ public class Vectorizer implements Physi
 
   private static class ValidatorVectorizationContext extends VectorizationContext {
     private ValidatorVectorizationContext() {
-      super();
+      super("No Name");
     }
 
     @Override
@@ -831,25 +823,27 @@ public class Vectorizer implements Physi
   }
 
   @Override
-  public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
-    this.physicalContext  = pctx;
-    boolean vectorPath = HiveConf.getBoolVar(pctx.getConf(),
+  public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
+    this.physicalContext  = physicalContext;
+    hiveConf = physicalContext.getConf();
+
+    boolean vectorPath = HiveConf.getBoolVar(hiveConf,
         HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
     if (!vectorPath) {
       LOG.info("Vectorization is disabled");
-      return pctx;
+      return physicalContext;
     }
     // create dispatcher and graph walker
-    Dispatcher disp = new VectorizationDispatcher(pctx);
+    Dispatcher disp = new VectorizationDispatcher(physicalContext);
     TaskGraphWalker ogw = new TaskGraphWalker(disp);
 
     // get all the tasks nodes from root task
     ArrayList<Node> topNodes = new ArrayList<Node>();
-    topNodes.addAll(pctx.getRootTasks());
+    topNodes.addAll(physicalContext.getRootTasks());
 
     // begin to walk through the task tree.
     ogw.startWalking(topNodes, null);
-    return pctx;
+    return physicalContext;
   }
 
   boolean validateMapWorkOperator(Operator<? extends OperatorDesc> op, MapWork mWork, boolean isTez) {
@@ -901,7 +895,7 @@ public class Vectorizer implements Physi
         }
         break;
       case GROUPBY:
-        if (HiveConf.getBoolVar(physicalContext.getConf(),
+        if (HiveConf.getBoolVar(hiveConf,
                     HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED)) {
           ret = validateGroupByOperator((GroupByOperator) op, true, true);
         } else {
@@ -1262,20 +1256,24 @@ public class Vectorizer implements Physi
     return supportedDataTypesPattern.matcher(type.toLowerCase()).matches();
   }
 
-  private VectorizationContext getVectorizationContext(Operator op,
-      PhysicalContext pctx) {
-    RowSchema rs = op.getSchema();
+  private VectorizationContext getVectorizationContext(RowSchema rowSchema, String contextName,
+    Map<Integer, String> typeNameMap) {
+
+    VectorizationContext vContext = new VectorizationContext(contextName);
 
     // Add all non-virtual columns to make a vectorization context for
     // the TableScan operator.
-    VectorizationContext vContext = new VectorizationContext();
-    for (ColumnInfo c : rs.getSignature()) {
+    int i = 0;
+    for (ColumnInfo c : rowSchema.getSignature()) {
       // Earlier, validation code should have eliminated virtual columns usage (HIVE-5560).
       if (!isVirtualColumn(c)) {
         vContext.addInitialColumn(c.getInternalName());
+        typeNameMap.put(i, c.getTypeName());
+        i++;
       }
     }
     vContext.finishedAddingInitialColumns();
+
     return vContext;
   }
 
@@ -1333,40 +1331,14 @@ public class Vectorizer implements Physi
     return false;
   }
 
-  public void debugDisplayAllMaps(Map<String, Map<String, Integer>> allColumnVectorMaps,
-          Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps) {
+  public void debugDisplayAllMaps(BaseWork work) {
 
-    // Context keys grow in length since they are a path...
-    Comparator<String> comparerShorterString = new Comparator<String>() {
-      @Override
-      public int compare(String o1, String o2) {
-        Integer length1 = o1.length();
-        Integer length2 = o2.length();
-        return length1.compareTo(length2);
-      }};
-
-    Comparator<Integer> comparerInteger = new Comparator<Integer>() {
-      @Override
-      public int compare(Integer o1, Integer o2) {
-        return o1.compareTo(o2);
-      }};
-
-    Map<String, Map<Integer, String>> sortedAllColumnVectorMaps = new TreeMap<String, Map<Integer, String>>(comparerShorterString);
-    for (Map.Entry<String, Map<String, Integer>> entry : allColumnVectorMaps.entrySet()) {
-      Map<Integer, String> sortedColumnMap = new TreeMap<Integer, String>(comparerInteger);
-      for (Map.Entry<String, Integer> innerEntry : entry.getValue().entrySet()) {
-        sortedColumnMap.put(innerEntry.getValue(), innerEntry.getKey());
-      }
-      sortedAllColumnVectorMaps.put(entry.getKey(), sortedColumnMap);
-    }
-    LOG.debug("sortedAllColumnVectorMaps " + sortedAllColumnVectorMaps);
-
-    Map<String, Map<Integer, String>> sortedAllScratchColumnVectorTypeMap = new TreeMap<String, Map<Integer, String>>(comparerShorterString);
-    for (Map.Entry<String, Map<Integer, String>> entry : allScratchColumnVectorTypeMaps.entrySet()) {
-      Map<Integer, String> sortedScratchColumnTypeMap = new TreeMap<Integer, String>(comparerInteger);
-      sortedScratchColumnTypeMap.putAll(entry.getValue());
-      sortedAllScratchColumnVectorTypeMap.put(entry.getKey(), sortedScratchColumnTypeMap);
-    }
-    LOG.debug("sortedAllScratchColumnVectorTypeMap " + sortedAllScratchColumnVectorTypeMap);
+    Map<String, Integer> columnNameMap = work.getVectorColumnNameMap();
+    Map<Integer, String> columnTypeMap = work.getVectorColumnTypeMap();
+    Map<Integer, String> scratchColumnTypeMap = work.getVectorScratchColumnTypeMap();
+
+    LOG.debug("debugDisplayAllMaps columnNameMap " + columnNameMap.toString());
+    LOG.debug("debugDisplayAllMaps columnTypeMap " + columnTypeMap.toString());
+    LOG.debug("debugDisplayAllMaps scratchColumnTypeMap " + scratchColumnTypeMap.toString());
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Tue Apr 14 01:59:29 2015
@@ -60,8 +60,9 @@ public abstract class BaseWork extends A
   private String name;
 
   // Vectorization.
-  protected Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = null;
-  protected Map<String, Map<String, Integer>> allColumnVectorMaps = null;
+  protected Map<String, Integer> vectorColumnNameMap;
+  protected Map<Integer, String> vectorColumnTypeMap;
+  protected Map<Integer, String> vectorScratchColumnTypeMap;
 
   public void setGatheringStats(boolean gatherStats) {
     this.gatheringStats = gatherStats;
@@ -143,21 +144,28 @@ public abstract class BaseWork extends A
     return returnSet;
   }
 
-  public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
-    return allScratchColumnVectorTypeMaps;
+  public Map<String, Integer> getVectorColumnNameMap() {
+    return vectorColumnNameMap;
   }
 
-  public void setAllScratchColumnVectorTypeMaps(
-      Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps) {
-    this.allScratchColumnVectorTypeMaps = allScratchColumnVectorTypeMaps;
+  public void setVectorColumnNameMap(Map<String, Integer> vectorColumnNameMap) {
+    this.vectorColumnNameMap = vectorColumnNameMap;
   }
 
-  public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
-    return allColumnVectorMaps;
+  public Map<Integer, String> getVectorColumnTypeMap() {
+    return vectorColumnTypeMap;
   }
 
-  public void setAllColumnVectorMaps(Map<String, Map<String, Integer>> allColumnVectorMaps) {
-    this.allColumnVectorMaps = allColumnVectorMaps;
+  public void setVectorColumnTypeMap(Map<Integer, String> vectorColumnTypeMap) {
+    this.vectorColumnTypeMap = vectorColumnTypeMap;
+  }
+
+  public Map<Integer, String> getVectorScratchColumnTypeMap() {
+    return vectorScratchColumnTypeMap;
+  }
+
+  public void setVectorScratchColumnTypeMap(Map<Integer, String> vectorScratchColumnTypeMap) {
+    this.vectorScratchColumnTypeMap = vectorScratchColumnTypeMap;
   }
 
   /**

Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/RandomRowObjectSource.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/RandomRowObjectSource.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/RandomRowObjectSource.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/RandomRowObjectSource.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,405 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalDayTimeObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalYearMonthObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hive.common.util.DateUtils;
+
+/**
+ * Generate object inspector and random row object[].
+ */
+public class RandomRowObjectSource {
+
+  private Random r;
+
+  private int columnCount;
+
+  private List<String> typeNames;
+
+  private PrimitiveCategory[] primitiveCategories;
+
+  private PrimitiveTypeInfo[] primitiveTypeInfos;
+
+  private List<ObjectInspector> primitiveObjectInspectorList;
+
+  private StructObjectInspector rowStructObjectInspector;
+
+  public List<String> typeNames() {
+    return typeNames;
+  }
+
+  public PrimitiveCategory[] primitiveCategories() {
+    return primitiveCategories;
+  }
+
+  public PrimitiveTypeInfo[] primitiveTypeInfos() {
+    return primitiveTypeInfos;
+  }
+
+  public StructObjectInspector rowStructObjectInspector() {
+    return rowStructObjectInspector;
+  }
+
+  public void init(Random r) {
+    this.r = r;
+    chooseSchema();
+  }
+
+  private static String[] possibleHiveTypeNames = {
+      "boolean",
+      "tinyint",
+      "smallint",
+      "int",
+      "bigint",
+      "date",
+      "float",
+      "double",
+      "string",
+      "char",
+      "varchar",
+      "binary",
+      "date",
+      "timestamp",
+      serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME,
+      serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME,
+      "decimal"
+  };
+
+  private void chooseSchema() {
+    columnCount = 1 + r.nextInt(20);
+    typeNames = new ArrayList<String>(columnCount);
+    primitiveCategories = new PrimitiveCategory[columnCount];
+    primitiveTypeInfos = new PrimitiveTypeInfo[columnCount];
+    primitiveObjectInspectorList = new ArrayList<ObjectInspector>(columnCount);
+    List<String> columnNames = new ArrayList<String>(columnCount);
+    for (int c = 0; c < columnCount; c++) {
+      columnNames.add(String.format("col%d", c));
+      int typeNum = r.nextInt(possibleHiveTypeNames.length);
+      String typeName = possibleHiveTypeNames[typeNum];
+      if (typeName.equals("char")) {
+        int maxLength = 1 + r.nextInt(100);
+        typeName = String.format("char(%d)", maxLength);
+      } else if (typeName.equals("varchar")) {
+        int maxLength = 1 + r.nextInt(100);
+        typeName = String.format("varchar(%d)", maxLength);
+      } else if (typeName.equals("decimal")) {
+        typeName = String.format("decimal(%d,%d)", HiveDecimal.SYSTEM_DEFAULT_PRECISION, HiveDecimal.SYSTEM_DEFAULT_SCALE);
+      }
+      PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+      primitiveTypeInfos[c] = primitiveTypeInfo;
+      PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
+      primitiveCategories[c] = primitiveCategory;
+      primitiveObjectInspectorList.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveTypeInfo));
+      typeNames.add(typeName);
+    }
+    rowStructObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, primitiveObjectInspectorList);
+  }
+
+  public Object[][] randomRows(int n) {
+    Object[][] result = new Object[n][];
+    for (int i = 0; i < n; i++) {
+      result[i] = randomRow();
+    }
+    return result;
+  }
+
+  public Object[] randomRow() {
+    Object row[] = new Object[columnCount];
+    for (int c = 0; c < columnCount; c++) {
+      Object object = randomObject(c);
+      if (object == null) {
+        throw new Error("Unexpected null for column " + c);
+      }
+      row[c] = getWritableObject(c, object);
+      if (row[c] == null) {
+        throw new Error("Unexpected null for writable for column " + c);
+      }
+    }
+    return row;
+  }
+
+  public Object getWritableObject(int column, Object object) {
+    ObjectInspector objectInspector = primitiveObjectInspectorList.get(column);
+    PrimitiveCategory primitiveCategory = primitiveCategories[column];
+    PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
+    switch (primitiveCategory) {
+    case BOOLEAN:
+      return ((WritableBooleanObjectInspector) objectInspector).create((boolean) object);
+    case BYTE:
+      return ((WritableByteObjectInspector) objectInspector).create((byte) object);
+    case SHORT:
+      return ((WritableShortObjectInspector) objectInspector).create((short) object);
+    case INT:
+      return ((WritableIntObjectInspector) objectInspector).create((int) object);
+    case LONG:
+      return ((WritableLongObjectInspector) objectInspector).create((long) object);
+    case DATE:
+      return ((WritableDateObjectInspector) objectInspector).create((Date) object);
+    case FLOAT:
+      return ((WritableFloatObjectInspector) objectInspector).create((float) object);
+    case DOUBLE:
+      return ((WritableDoubleObjectInspector) objectInspector).create((double) object);
+    case STRING:
+      return ((WritableStringObjectInspector) objectInspector).create((String) object);
+    case CHAR:
+      {
+        WritableHiveCharObjectInspector writableCharObjectInspector = 
+                new WritableHiveCharObjectInspector( (CharTypeInfo) primitiveTypeInfo);
+        return writableCharObjectInspector.create(new HiveChar(StringUtils.EMPTY, -1));
+      }
+    case VARCHAR:
+      {
+        WritableHiveVarcharObjectInspector writableVarcharObjectInspector = 
+                new WritableHiveVarcharObjectInspector( (VarcharTypeInfo) primitiveTypeInfo);
+        return writableVarcharObjectInspector.create(new HiveVarchar(StringUtils.EMPTY, -1));
+      }
+    case BINARY:
+      return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create(ArrayUtils.EMPTY_BYTE_ARRAY);
+    case TIMESTAMP:
+      return ((WritableTimestampObjectInspector) objectInspector).create(new Timestamp(0));
+    case INTERVAL_YEAR_MONTH:
+      return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create(new HiveIntervalYearMonth(0));
+    case INTERVAL_DAY_TIME:
+      return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create(new HiveIntervalDayTime(0, 0));
+    case DECIMAL:
+      {
+        WritableHiveDecimalObjectInspector writableDecimalObjectInspector =
+                new WritableHiveDecimalObjectInspector((DecimalTypeInfo) primitiveTypeInfo);
+        return writableDecimalObjectInspector.create(HiveDecimal.ZERO);
+      }
+    default:
+      throw new Error("Unknown primitive category " + primitiveCategory);
+    }
+  }
+
+  public Object randomObject(int column) {
+    PrimitiveCategory primitiveCategory = primitiveCategories[column];
+    PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
+    switch (primitiveCategory) {
+    case BOOLEAN:
+      return Boolean.valueOf(r.nextInt(1) == 1);
+    case BYTE:
+      return Byte.valueOf((byte) r.nextInt());
+    case SHORT:
+      return Short.valueOf((short) r.nextInt());
+    case INT:
+      return Integer.valueOf(r.nextInt());
+    case LONG:
+      return Long.valueOf(r.nextLong());
+    case DATE:
+      return getRandDate(r);
+    case FLOAT:
+      return Float.valueOf(r.nextFloat() * 10 - 5);
+    case DOUBLE:
+      return Double.valueOf(r.nextDouble() * 10 - 5);
+    case STRING:
+      return getRandString(r);
+    case CHAR:
+      return getRandHiveChar(r, (CharTypeInfo) primitiveTypeInfo);
+    case VARCHAR:
+      return getRandHiveVarchar(r, (VarcharTypeInfo) primitiveTypeInfo);
+    case BINARY:
+      return getRandBinary(r, 1 + r.nextInt(100));
+    case TIMESTAMP:
+      return getRandTimestamp(r);
+    case INTERVAL_YEAR_MONTH:
+      return getRandIntervalYearMonth(r);
+    case INTERVAL_DAY_TIME:
+      return getRandIntervalDayTime(r);
+    case DECIMAL:
+      return getRandHiveDecimal(r, (DecimalTypeInfo) primitiveTypeInfo);
+    default:
+      throw new Error("Unknown primitive category " + primitiveCategory);
+    }
+  }
+
+  public static String getRandString(Random r) {
+    return getRandString(r, null, r.nextInt(10));
+  }
+
+  public static String getRandString(Random r, String characters, int length) {
+    if (characters == null) {
+      characters = "ABCDEFGHIJKLMabcdefghijklm";
+    }
+    StringBuilder sb = new StringBuilder();
+    sb.append("");
+    for (int i = 0; i < length; i++) {
+      if (characters == null) {
+        sb.append((char) (r.nextInt(128)));
+      } else {
+        sb.append(characters.charAt(r.nextInt(characters.length())));
+      }
+    }
+    return sb.toString();
+  }
+
+  public static HiveChar getRandHiveChar(Random r, CharTypeInfo charTypeInfo) {
+    int maxLength = 1 + r.nextInt(charTypeInfo.getLength());
+    String randomString = getRandString(r, "abcdefghijklmnopqrstuvwxyz", 100);
+    HiveChar hiveChar = new HiveChar(randomString, maxLength);
+    return hiveChar;
+  }
+
+  public static HiveVarchar getRandHiveVarchar(Random r, VarcharTypeInfo varcharTypeInfo) {
+    int maxLength = 1 + r.nextInt(varcharTypeInfo.getLength());
+    String randomString = getRandString(r, "abcdefghijklmnopqrstuvwxyz", 100);
+    HiveVarchar hiveVarchar = new HiveVarchar(randomString, maxLength);
+    return hiveVarchar;
+  }
+
+  public static byte[] getRandBinary(Random r, int len){
+    byte[] bytes = new byte[len];
+    for (int j = 0; j < len; j++){
+      bytes[j] = Byte.valueOf((byte) r.nextInt());
+    }
+    return bytes;
+  }
+
+  private static final String DECIMAL_CHARS = "0123456789";
+
+  public static HiveDecimal getRandHiveDecimal(Random r, DecimalTypeInfo decimalTypeInfo) {
+    while (true) {
+      StringBuilder sb = new StringBuilder();
+      int precision = 1 + r.nextInt(18);
+      int scale = 0 + r.nextInt(precision + 1);
+
+      int integerDigits = precision - scale;
+
+      if (r.nextBoolean()) {
+        sb.append("-");
+      }
+
+      if (integerDigits == 0) {
+        sb.append("0");
+      } else {
+        sb.append(getRandString(r, DECIMAL_CHARS, integerDigits));
+      }
+      if (scale != 0) {
+        sb.append(".");
+        sb.append(getRandString(r, DECIMAL_CHARS, scale));
+      }
+
+      HiveDecimal bd = HiveDecimal.create(sb.toString());
+      if (bd.scale() > bd.precision()) {
+        // Sometimes weird decimals are produced?
+        continue;
+      }
+
+      return bd;
+    }
+  }
+
+  public static Date getRandDate(Random r) {
+    String dateStr = String.format("%d-%02d-%02d",
+        Integer.valueOf(1800 + r.nextInt(500)),  // year
+        Integer.valueOf(1 + r.nextInt(12)),      // month
+        Integer.valueOf(1 + r.nextInt(28)));     // day
+    Date dateVal = Date.valueOf(dateStr);
+    return dateVal;
+  }
+
+  public static Timestamp getRandTimestamp(Random r) {
+    String optionalNanos = "";
+    if (r.nextInt(2) == 1) {
+      optionalNanos = String.format(".%09d",
+          Integer.valueOf(0 + r.nextInt(DateUtils.NANOS_PER_SEC)));
+    }
+    String timestampStr = String.format("%d-%02d-%02d %02d:%02d:%02d%s",
+        Integer.valueOf(1970 + r.nextInt(200)),  // year
+        Integer.valueOf(1 + r.nextInt(12)),      // month
+        Integer.valueOf(1 + r.nextInt(28)),      // day
+        Integer.valueOf(0 + r.nextInt(24)),      // hour
+        Integer.valueOf(0 + r.nextInt(60)),      // minute
+        Integer.valueOf(0 + r.nextInt(60)),      // second
+        optionalNanos);
+    Timestamp timestampVal = Timestamp.valueOf(timestampStr);
+    return timestampVal;
+  }
+
+  public static HiveIntervalYearMonth getRandIntervalYearMonth(Random r) {
+    String yearMonthSignStr = r.nextInt(2) == 0 ? "" : "-";
+    String intervalYearMonthStr = String.format("%s%d-%d",
+        yearMonthSignStr,
+        Integer.valueOf(1800 + r.nextInt(500)),  // year
+        Integer.valueOf(0 + r.nextInt(12)));     // month
+    HiveIntervalYearMonth intervalYearMonthVal = HiveIntervalYearMonth.valueOf(intervalYearMonthStr);
+    TestCase.assertTrue(intervalYearMonthVal != null);
+    return intervalYearMonthVal;
+  }
+
+  public static HiveIntervalDayTime getRandIntervalDayTime(Random r) {
+    String optionalNanos = "";
+    if (r.nextInt(2) == 1) {
+      optionalNanos = String.format(".%09d",
+          Integer.valueOf(0 + r.nextInt(DateUtils.NANOS_PER_SEC)));
+    }
+    String yearMonthSignStr = r.nextInt(2) == 0 ? "" : "-";
+    String dayTimeStr = String.format("%s%d %02d:%02d:%02d%s",
+        yearMonthSignStr,
+        Integer.valueOf(1 + r.nextInt(28)),      // day
+        Integer.valueOf(0 + r.nextInt(24)),      // hour
+        Integer.valueOf(0 + r.nextInt(60)),      // minute
+        Integer.valueOf(0 + r.nextInt(60)),      // second
+        optionalNanos);
+    HiveIntervalDayTime intervalDayTimeVal = HiveIntervalDayTime.valueOf(dayTimeStr);
+    TestCase.assertTrue(intervalDayTimeVal != null);
+    return intervalDayTimeVal;
+  }
+}

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java Tue Apr 14 01:59:29 2015
@@ -88,7 +88,7 @@ public class TestVectorFilterOperator {
     ExprNodeColumnDesc col1Expr = new  ExprNodeColumnDesc(Long.class, "col1", "table", false);
     List<String> columns = new ArrayList<String>();
     columns.add("col1");
-    VectorizationContext vc = new VectorizationContext(columns);
+    VectorizationContext vc = new VectorizationContext("name", columns);
     FilterDesc fdesc = new FilterDesc();
     fdesc.setPredicate(col1Expr);
     return new VectorFilterOperator(vc, fdesc);

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java Tue Apr 14 01:59:29 2015
@@ -173,7 +173,7 @@ public class TestVectorGroupByOperator {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("Key");
     mapColumnNames.add("Value");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildKeyGroupByDesc (ctx, "max",
         "Value", TypeInfoFactory.longTypeInfo,
@@ -1710,7 +1710,7 @@ public class TestVectorGroupByOperator {
 
     mapColumnNames.put("value", i);
     outputColumnNames.add("value");
-    VectorizationContext ctx = new VectorizationContext(outputColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", outputColumnNames);
 
     ArrayList<AggregationDesc> aggs = new ArrayList(1);
     aggs.add(
@@ -1821,7 +1821,7 @@ public class TestVectorGroupByOperator {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("Key");
     mapColumnNames.add("Value");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
     Set<Object> keys = new HashSet<Object>();
 
     AggregationDesc agg = buildAggregationDesc(ctx, aggregateName,
@@ -2235,7 +2235,7 @@ public class TestVectorGroupByOperator {
       Object expected) throws HiveException {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildGroupByDescCountStar (ctx);
 
@@ -2264,7 +2264,7 @@ public class TestVectorGroupByOperator {
       Object expected) throws HiveException {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildGroupByDescType(ctx, "count", "A", TypeInfoFactory.longTypeInfo);
     VectorGroupByDesc vectorDesc = desc.getVectorDesc();
@@ -2296,7 +2296,7 @@ public class TestVectorGroupByOperator {
       Object expected) throws HiveException {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A",
         TypeInfoFactory.stringTypeInfo);
@@ -2322,11 +2322,12 @@ public class TestVectorGroupByOperator {
   }
 
   public void testAggregateDecimalIterable (
-String aggregateName, Iterable<VectorizedRowBatch> data,
-      Object expected) throws HiveException {
-    List<String> mapColumnNames = new ArrayList<String>();
-    mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+          String aggregateName,
+          Iterable<VectorizedRowBatch> data,
+          Object expected) throws HiveException {
+          List<String> mapColumnNames = new ArrayList<String>();
+          mapColumnNames.add("A");
+          VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc =
         buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4));
@@ -2358,7 +2359,7 @@ String aggregateName, Iterable<Vectorize
       Object expected) throws HiveException {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildGroupByDescType (ctx, aggregateName, "A",
         TypeInfoFactory.doubleTypeInfo);
@@ -2389,7 +2390,7 @@ String aggregateName, Iterable<Vectorize
       Object expected) throws HiveException {
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("A");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.longTypeInfo);
 
@@ -2420,7 +2421,7 @@ String aggregateName, Iterable<Vectorize
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("Key");
     mapColumnNames.add("Value");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
 
     Set<Object> keys = new HashSet<Object>();
 
@@ -2487,7 +2488,7 @@ String aggregateName, Iterable<Vectorize
     List<String> mapColumnNames = new ArrayList<String>();
     mapColumnNames.add("Key");
     mapColumnNames.add("Value");
-    VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+    VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
     Set<Object> keys = new HashSet<Object>();
 
     GroupByDesc desc = buildKeyGroupByDesc (ctx, aggregateName, "Value",

Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit test for the vectorized conversion to and from row object[].
+ */
+public class TestVectorRowObject extends TestCase {
+
+  void examineBatch(VectorizedRowBatch batch, VectorExtractRowSameBatch vectorExtractRow,
+              Object[][] randomRows, int firstRandomRowIndex ) {
+
+    int rowSize = vectorExtractRow.getCount();
+    Object[] row = new Object[rowSize];
+    for (int i = 0; i < batch.size; i++) {
+      vectorExtractRow.extractRow(i, row);
+      Object[] expectedRow = randomRows[firstRandomRowIndex + i];
+      for (int c = 0; c < rowSize; c++) {
+        if (!row[c].equals(expectedRow[c])) {
+          fail("Row " + (firstRandomRowIndex + i) + " and column " + c + " mismatch");
+        }
+      }
+    }
+  }
+
+  void testVectorRowObject(int caseNum, Random r) throws HiveException {
+
+    Map<Integer, String> emptyScratchMap = new HashMap<Integer, String>();
+
+    RandomRowObjectSource source = new RandomRowObjectSource();
+    source.init(r);
+
+    VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
+    batchContext.init(emptyScratchMap, source.rowStructObjectInspector());
+    VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+
+    VectorAssignRowSameBatch vectorAssignRow = new VectorAssignRowSameBatch();
+    vectorAssignRow.init(source.typeNames());
+    vectorAssignRow.setOneBatch(batch);
+    
+    VectorExtractRowSameBatch vectorExtractRow = new VectorExtractRowSameBatch();
+    vectorExtractRow.init(source.typeNames());
+    vectorExtractRow.setOneBatch(batch);
+
+    Object[][] randomRows = source.randomRows(100000);
+    int firstRandomRowIndex = 0;
+    for (int i = 0; i < randomRows.length; i++) {
+      Object[] row = randomRows[i];
+
+      vectorAssignRow.assignRow(batch.size, row);
+      batch.size++;
+      if (batch.size == batch.DEFAULT_SIZE) {
+        examineBatch(batch, vectorExtractRow, randomRows, firstRandomRowIndex);
+        firstRandomRowIndex = i + 1;
+        batch.reset();
+      }
+    }
+    if (batch.size > 0) {
+      examineBatch(batch, vectorExtractRow, randomRows, firstRandomRowIndex);
+    }
+  }
+
+  public void testVectorRowObject() throws Throwable {
+
+  try {
+    Random r = new Random(5678);
+    for (int c = 0; c < 10; c++) {
+      testVectorRowObject(c, r);
+    }
+  } catch (Throwable e) {
+    e.printStackTrace();
+    throw e;
+  }
+  }
+}
\ No newline at end of file

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java Tue Apr 14 01:59:29 2015
@@ -88,7 +88,7 @@ public class TestVectorSelectOperator {
     columns.add("a");
     columns.add("b");
     columns.add("c");
-    VectorizationContext vc = new VectorizationContext(columns);
+    VectorizationContext vc = new VectorizationContext("name", columns);
 
     SelectDesc selDesc = new SelectDesc(false);
     List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();

Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,658 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.io.IOException;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite;
+import org.apache.hadoop.hive.serde2.fast.DeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.apache.hadoop.hive.serde2.fast.SerializeWrite;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit test for the vectorized serialize and deserialize row.
+ */
+public class TestVectorSerDeRow extends TestCase {
+
+  public static enum SerializationType {
+    NONE,
+    BINARY_SORTABLE,
+    LAZY_BINARY,
+    LAZY_SIMPLE
+  }
+
+  void deserializeAndVerify(Output output, DeserializeRead deserializeRead, 
+              RandomRowObjectSource source, Object[] expectedRow)
+              throws HiveException, IOException {
+    deserializeRead.set(output.getData(),  0, output.getLength());
+    PrimitiveCategory[] primitiveCategories = source.primitiveCategories();
+    for (int i = 0; i < primitiveCategories.length; i++) {
+      Object expected = expectedRow[i];
+      PrimitiveCategory primitiveCategory = primitiveCategories[i];
+      PrimitiveTypeInfo primitiveTypeInfo = source.primitiveTypeInfos()[i];
+      if (deserializeRead.readCheckNull()) {
+        throw new HiveException("Unexpected NULL");
+      }
+      switch (primitiveCategory) {
+      case BOOLEAN:
+        {
+          Boolean value = deserializeRead.readBoolean();
+          BooleanWritable expectedWritable = (BooleanWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case BYTE:
+        {
+          Byte value = deserializeRead.readByte();
+          ByteWritable expectedWritable = (ByteWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
+          }
+        }
+        break;
+      case SHORT:
+        {
+          Short value = deserializeRead.readShort();
+          ShortWritable expectedWritable = (ShortWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case INT:
+        {
+          Integer value = deserializeRead.readInt();
+          IntWritable expectedWritable = (IntWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case LONG:
+        {
+          Long value = deserializeRead.readLong();
+          LongWritable expectedWritable = (LongWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case DATE:
+        {
+          DeserializeRead.ReadDateResults readDateResults = deserializeRead.createReadDateResults();
+          deserializeRead.readDate(readDateResults);
+          Date value = readDateResults.getDate();
+          DateWritable expectedWritable = (DateWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
+          }
+        }
+        break;
+      case FLOAT:
+        {
+          Float value = deserializeRead.readFloat();
+          FloatWritable expectedWritable = (FloatWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case DOUBLE:
+        {
+          Double value = deserializeRead.readDouble();
+          DoubleWritable expectedWritable = (DoubleWritable) expected;
+          if (!value.equals(expectedWritable.get())) {
+            TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
+          }
+        }
+        break;
+      case STRING:
+        {
+          DeserializeRead.ReadStringResults readStringResults = deserializeRead.createReadStringResults();
+          deserializeRead.readString(readStringResults);
+
+          char[] charsBuffer = new char[readStringResults.bytes.length];
+          for (int c = 0; c < charsBuffer.length; c++) {
+            charsBuffer[c] = (char) (readStringResults.bytes[c] & 0xFF);
+          }
+
+          byte[] stringBytes = Arrays.copyOfRange(readStringResults.bytes, readStringResults.start, readStringResults.start + readStringResults.length);
+
+          char[] charsRange = new char[stringBytes.length];
+          for (int c = 0; c < charsRange.length; c++) {
+            charsRange[c] = (char) (stringBytes[c] & 0xFF);
+          }
+
+          Text text = new Text(stringBytes);
+          String value = text.toString();
+          Text expectedWritable = (Text) expected;
+          if (!value.equals(expectedWritable.toString())) {
+            TestCase.fail("String field mismatch (expected '" + expectedWritable.toString() + "' found '" + value + "')");
+          }
+        }
+        break;
+      case CHAR:
+        {
+          DeserializeRead.ReadHiveCharResults readHiveCharResults = deserializeRead.createReadHiveCharResults();
+          deserializeRead.readHiveChar(readHiveCharResults);
+          HiveChar hiveChar = readHiveCharResults.getHiveChar();
+          HiveCharWritable expectedWritable = (HiveCharWritable) expected;
+          if (!hiveChar.equals(expectedWritable.getHiveChar())) {
+            TestCase.fail("Char field mismatch (expected '" + expectedWritable.getHiveChar() + "' found '" + hiveChar + "')");
+          }
+        }
+        break;
+      case VARCHAR:
+        {
+          DeserializeRead.ReadHiveVarcharResults readHiveVarcharResults = deserializeRead.createReadHiveVarcharResults();
+          deserializeRead.readHiveVarchar(readHiveVarcharResults);
+          HiveVarchar hiveVarchar = readHiveVarcharResults.getHiveVarchar();
+          HiveVarcharWritable expectedWritable = (HiveVarcharWritable) expected;
+          if (!hiveVarchar.equals(expectedWritable.getHiveVarchar())) {
+            TestCase.fail("Varchar field mismatch (expected '" + expectedWritable.getHiveVarchar() + "' found '" + hiveVarchar + "')");
+          }
+        }
+        break;
+      case DECIMAL:
+        {
+          DeserializeRead.ReadDecimalResults readDecimalResults = deserializeRead.createReadDecimalResults();
+          deserializeRead.readHiveDecimal(readDecimalResults);
+          HiveDecimal value = readDecimalResults.getHiveDecimal();
+          if (value == null) {
+            TestCase.fail("Decimal field evaluated to NULL");
+          }
+          HiveDecimalWritable expectedWritable = (HiveDecimalWritable) expected;
+          if (!value.equals(expectedWritable.getHiveDecimal())) {
+            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
+            int precision = decimalTypeInfo.getPrecision();
+            int scale = decimalTypeInfo.getScale();
+            TestCase.fail("Decimal field mismatch (expected " + expectedWritable.getHiveDecimal() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
+          }
+        }
+        break;
+    case TIMESTAMP:
+      {
+        DeserializeRead.ReadTimestampResults readTimestampResults = deserializeRead.createReadTimestampResults();
+        deserializeRead.readTimestamp(readTimestampResults);
+        Timestamp value = readTimestampResults.getTimestamp();
+        TimestampWritable expectedWritable = (TimestampWritable) expected;
+        if (!value.equals(expectedWritable.getTimestamp())) {
+          TestCase.fail("Timestamp field mismatch (expected " + expectedWritable.getTimestamp() + " found " + value.toString() + ")");
+        }
+      }
+      break;
+    case INTERVAL_YEAR_MONTH:
+      {
+        DeserializeRead.ReadIntervalYearMonthResults readIntervalYearMonthResults = deserializeRead.createReadIntervalYearMonthResults();
+        deserializeRead.readIntervalYearMonth(readIntervalYearMonthResults);
+        HiveIntervalYearMonth value = readIntervalYearMonthResults.getHiveIntervalYearMonth();
+        HiveIntervalYearMonthWritable expectedWritable = (HiveIntervalYearMonthWritable) expected;
+        HiveIntervalYearMonth expectedValue = expectedWritable.getHiveIntervalYearMonth();
+        if (!value.equals(expectedValue)) {
+          TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
+        }
+      }
+      break;
+    case INTERVAL_DAY_TIME:
+      {
+        DeserializeRead.ReadIntervalDayTimeResults readIntervalDayTimeResults = deserializeRead.createReadIntervalDayTimeResults();
+        deserializeRead.readIntervalDayTime(readIntervalDayTimeResults);
+        HiveIntervalDayTime value = readIntervalDayTimeResults.getHiveIntervalDayTime();
+        HiveIntervalDayTimeWritable expectedWritable = (HiveIntervalDayTimeWritable) expected;
+        HiveIntervalDayTime expectedValue = expectedWritable.getHiveIntervalDayTime();
+        if (!value.equals(expectedValue)) {
+          TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
+        }
+      }
+      break;
+    case BINARY:
+      {
+        DeserializeRead.ReadBinaryResults readBinaryResults = deserializeRead.createReadBinaryResults();
+        deserializeRead.readBinary(readBinaryResults);
+        byte[] byteArray = Arrays.copyOfRange(readBinaryResults.bytes, readBinaryResults.start, readBinaryResults.start + readBinaryResults.length);
+        BytesWritable expectedWritable = (BytesWritable) expected;
+        if (byteArray.length != expectedWritable.getLength()){
+          TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + byteArray + ")");
+        }
+        byte[] expectedBytes = expectedWritable.getBytes();
+        for (int b = 0; b < byteArray.length; b++) {
+          if (byteArray[b] != expectedBytes[b]) {
+            TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + byteArray + ")");
+          }
+        }
+      }
+      break;
+      default:
+        throw new HiveException("Unexpected primitive category " + primitiveCategory);
+      }
+    }
+    deserializeRead.extraFieldsCheck();
+    TestCase.assertTrue(!deserializeRead.readBeyondConfiguredFieldsWarned());
+    TestCase.assertTrue(!deserializeRead.readBeyondBufferRangeWarned());
+    TestCase.assertTrue(!deserializeRead.bufferRangeHasExtraDataWarned());
+  }
+
+  void serializeBatch(VectorizedRowBatch batch, VectorSerializeRow vectorSerializeRow,
+           DeserializeRead deserializeRead, RandomRowObjectSource source, Object[][] randomRows,
+           int firstRandomRowIndex) throws HiveException, IOException {
+
+    Output output = new Output();
+    for (int i = 0; i < batch.size; i++) {
+      output.reset();
+      vectorSerializeRow.setOutput(output);
+      vectorSerializeRow.serializeWrite(batch, i);
+      Object[] expectedRow = randomRows[firstRandomRowIndex + i];
+
+      byte[] bytes = output.getData();
+      int length = output.getLength();
+      char[] chars = new char[length];
+      for (int c = 0; c < chars.length; c++) {
+        chars[c] = (char) (bytes[c] & 0xFF);
+      }
+
+      deserializeAndVerify(output, deserializeRead, source, expectedRow);
+    }
+  }
+
+  void testVectorSerializeRow(int caseNum, Random r, SerializationType serializationType) throws HiveException, IOException, SerDeException {
+
+    Map<Integer, String> emptyScratchMap = new HashMap<Integer, String>();
+
+    RandomRowObjectSource source = new RandomRowObjectSource();
+    source.init(r);
+
+    VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
+    batchContext.init(emptyScratchMap, source.rowStructObjectInspector());
+    VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+
+    VectorAssignRowSameBatch vectorAssignRow = new VectorAssignRowSameBatch();
+    vectorAssignRow.init(source.typeNames());
+    vectorAssignRow.setOneBatch(batch);
+
+    int fieldCount = source.typeNames().size();
+    DeserializeRead deserializeRead;
+    SerializeWrite serializeWrite;
+    switch (serializationType) {
+    case BINARY_SORTABLE:
+      deserializeRead = new BinarySortableDeserializeRead(source.primitiveTypeInfos());
+      serializeWrite = new BinarySortableSerializeWrite(fieldCount);
+      break;
+    case LAZY_BINARY:
+      deserializeRead = new LazyBinaryDeserializeRead(source.primitiveTypeInfos());
+      serializeWrite = new LazyBinarySerializeWrite(fieldCount);
+      break;
+    case LAZY_SIMPLE:
+      {
+        StructObjectInspector rowObjectInspector = source.rowStructObjectInspector();
+        LazySerDeParameters lazySerDeParams = getSerDeParams(rowObjectInspector);
+        byte separator = (byte) '\t';
+        deserializeRead = new LazySimpleDeserializeRead(source.primitiveTypeInfos(),
+            separator, lazySerDeParams);
+        serializeWrite = new LazySimpleSerializeWrite(fieldCount,
+            separator, lazySerDeParams);
+      }
+      break;
+    default:
+      throw new Error("Unknown serialization type " + serializationType);
+    }
+    VectorSerializeRow vectorSerializeRow = new VectorSerializeRow(serializeWrite);
+    vectorSerializeRow.init(source.typeNames());
+
+    Object[][] randomRows = source.randomRows(100000);
+    int firstRandomRowIndex = 0;
+    for (int i = 0; i < randomRows.length; i++) {
+      Object[] row = randomRows[i];
+
+      vectorAssignRow.assignRow(batch.size, row);
+      batch.size++;
+      if (batch.size == batch.DEFAULT_SIZE) {
+        serializeBatch(batch, vectorSerializeRow, deserializeRead, source, randomRows, firstRandomRowIndex);
+        firstRandomRowIndex = i + 1;
+        batch.reset();
+      }
+    }
+    if (batch.size > 0) {
+      serializeBatch(batch, vectorSerializeRow, deserializeRead, source, randomRows, firstRandomRowIndex);
+    }
+  }
+
+  void examineBatch(VectorizedRowBatch batch, VectorExtractRowSameBatch vectorExtractRow,
+      Object[][] randomRows, int firstRandomRowIndex ) {
+
+    int rowSize = vectorExtractRow.getCount();
+    Object[] row = new Object[rowSize];
+    for (int i = 0; i < batch.size; i++) {
+      vectorExtractRow.extractRow(i, row);
+
+      Object[] expectedRow = randomRows[firstRandomRowIndex + i];
+
+      for (int c = 0; c < rowSize; c++) {
+        if (row[c] == null) {
+          fail("Unexpected NULL from extractRow");
+        }
+        if (!row[c].equals(expectedRow[c])) {
+          fail("Row " + (firstRandomRowIndex + i) + " and column " + c + " mismatch");
+        }
+      }
+    }
+  }
+
+  private Output serializeRow(Object[] row, RandomRowObjectSource source, SerializeWrite serializeWrite) throws HiveException, IOException {
+    Output output = new Output();
+    serializeWrite.set(output);
+    PrimitiveCategory[] primitiveCategories = source.primitiveCategories();
+    for (int i = 0; i < primitiveCategories.length; i++) {
+      Object object = row[i];
+      PrimitiveCategory primitiveCategory = primitiveCategories[i];
+      switch (primitiveCategory) {
+      case BOOLEAN:
+        {
+          BooleanWritable expectedWritable = (BooleanWritable) object;
+          boolean value = expectedWritable.get();
+          serializeWrite.writeBoolean(value);
+        }
+        break;
+      case BYTE:
+        {
+          ByteWritable expectedWritable = (ByteWritable) object;
+          byte value = expectedWritable.get();
+          serializeWrite.writeByte(value);
+        }
+        break;
+      case SHORT:
+        {
+          ShortWritable expectedWritable = (ShortWritable) object;
+          short value = expectedWritable.get();
+          serializeWrite.writeShort(value);
+        }
+        break;
+      case INT:
+        {
+          IntWritable expectedWritable = (IntWritable) object;
+          int value = expectedWritable.get();
+          serializeWrite.writeInt(value);
+        }
+        break;
+      case LONG:
+        {
+          LongWritable expectedWritable = (LongWritable) object;
+          long value = expectedWritable.get();
+          serializeWrite.writeLong(value);
+          }
+        break;
+      case DATE:
+        {
+          DateWritable expectedWritable = (DateWritable) object;
+          Date value = expectedWritable.get();
+          serializeWrite.writeDate(value);
+        }
+        break;
+      case FLOAT:
+        {
+          FloatWritable expectedWritable = (FloatWritable) object;
+          float value = expectedWritable.get();
+          serializeWrite.writeFloat(value);
+        }
+        break;
+      case DOUBLE:
+        {
+          DoubleWritable expectedWritable = (DoubleWritable) object;
+          double value = expectedWritable.get();
+          serializeWrite.writeDouble(value);
+        }
+        break;
+      case STRING:
+        {
+          Text text = (Text) object;
+          serializeWrite.writeString(text.getBytes(), 0, text.getLength());
+        }
+        break;
+      case CHAR:
+        {
+          HiveCharWritable expectedWritable = (HiveCharWritable) object;
+          HiveChar value = expectedWritable.getHiveChar();
+          serializeWrite.writeHiveChar(value);
+        }
+        break;
+      case VARCHAR:
+        {
+          HiveVarcharWritable expectedWritable = (HiveVarcharWritable) object;
+          HiveVarchar value = expectedWritable.getHiveVarchar();
+          serializeWrite.writeHiveVarchar(value);
+        }
+        break;
+      case BINARY:
+        {
+          BytesWritable expectedWritable = (BytesWritable) object;
+          byte[] bytes = expectedWritable.getBytes();
+          int length = expectedWritable.getLength();
+          serializeWrite.writeBinary(bytes, 0, length);
+        }
+        break;
+      case TIMESTAMP:
+        {
+          TimestampWritable expectedWritable = (TimestampWritable) object;
+          Timestamp value = expectedWritable.getTimestamp();
+          serializeWrite.writeTimestamp(value);
+        }
+        break;
+      case INTERVAL_YEAR_MONTH:
+        {
+          HiveIntervalYearMonthWritable expectedWritable = (HiveIntervalYearMonthWritable) object;
+          HiveIntervalYearMonth value = expectedWritable.getHiveIntervalYearMonth();
+          serializeWrite.writeHiveIntervalYearMonth(value);
+        }
+        break;
+      case INTERVAL_DAY_TIME:
+        {
+          HiveIntervalDayTimeWritable expectedWritable = (HiveIntervalDayTimeWritable) object;
+          HiveIntervalDayTime value = expectedWritable.getHiveIntervalDayTime();
+          serializeWrite.writeHiveIntervalDayTime(value);
+        }
+        break;
+      case DECIMAL:
+        {
+          HiveDecimalWritable expectedWritable = (HiveDecimalWritable) object;
+          HiveDecimal value = expectedWritable.getHiveDecimal();
+          serializeWrite.writeHiveDecimal(value);
+        }
+        break;
+      default:
+        throw new HiveException("Unexpected primitive category " + primitiveCategory);
+      }
+    }
+    return output;
+  }
+
+  private Properties createProperties(String fieldNames, String fieldTypes) {
+    Properties tbl = new Properties();
+
+    // Set the configuration parameters
+    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
+    
+    tbl.setProperty("columns", fieldNames);
+    tbl.setProperty("columns.types", fieldTypes);
+
+    tbl.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
+
+    return tbl;
+  }
+
+  private LazySerDeParameters getSerDeParams(StructObjectInspector rowObjectInspector) throws SerDeException {
+    String fieldNames = ObjectInspectorUtils.getFieldNames(rowObjectInspector);
+    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowObjectInspector);
+    Configuration conf = new Configuration();
+    Properties tbl = createProperties(fieldNames, fieldTypes);
+    return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+  }
+
+  void testVectorDeserializeRow(int caseNum, Random r, SerializationType serializationType) throws HiveException, IOException, SerDeException {
+
+    Map<Integer, String> emptyScratchMap = new HashMap<Integer, String>();
+
+    RandomRowObjectSource source = new RandomRowObjectSource();
+    source.init(r);
+
+    VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
+    batchContext.init(emptyScratchMap, source.rowStructObjectInspector());
+    VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+
+    int fieldCount = source.typeNames().size();
+    DeserializeRead deserializeRead;
+    SerializeWrite serializeWrite;
+    switch (serializationType) {
+    case BINARY_SORTABLE:
+      deserializeRead = new BinarySortableDeserializeRead(source.primitiveTypeInfos());
+      serializeWrite = new BinarySortableSerializeWrite(fieldCount);
+      break;
+    case LAZY_BINARY:
+      deserializeRead = new LazyBinaryDeserializeRead(source.primitiveTypeInfos());
+      serializeWrite = new LazyBinarySerializeWrite(fieldCount);
+      break;
+    case LAZY_SIMPLE:
+      {
+        StructObjectInspector rowObjectInspector = source.rowStructObjectInspector();
+        LazySerDeParameters lazySerDeParams = getSerDeParams(rowObjectInspector);
+        byte separator = (byte) '\t';
+        deserializeRead = new LazySimpleDeserializeRead(source.primitiveTypeInfos(),
+            separator, lazySerDeParams);
+        serializeWrite = new LazySimpleSerializeWrite(fieldCount,
+            separator, lazySerDeParams);
+      }
+      break;
+    default:
+      throw new Error("Unknown serialization type " + serializationType);
+    }
+    VectorDeserializeRow vectorDeserializeRow = new VectorDeserializeRow(deserializeRead);
+    vectorDeserializeRow.init();
+
+    VectorExtractRowSameBatch vectorExtractRow = new VectorExtractRowSameBatch();
+    vectorExtractRow.init(source.typeNames());
+    vectorExtractRow.setOneBatch(batch);
+
+    Object[][] randomRows = source.randomRows(100000);
+    int firstRandomRowIndex = 0;
+    for (int i = 0; i < randomRows.length; i++) {
+      Object[] row = randomRows[i];
+
+      Output output = serializeRow(row, source, serializeWrite);
+      vectorDeserializeRow.setBytes(output.getData(), 0, output.getLength());
+      vectorDeserializeRow.deserializeByValue(batch, batch.size);
+      batch.size++;
+      if (batch.size == batch.DEFAULT_SIZE) {
+        examineBatch(batch, vectorExtractRow, randomRows, firstRandomRowIndex);
+        firstRandomRowIndex = i + 1;
+        batch.reset();
+      }
+    }
+    if (batch.size > 0) {
+      examineBatch(batch, vectorExtractRow, randomRows, firstRandomRowIndex);
+    }
+  }
+
+  public void testVectorSerDeRow() throws Throwable {
+
+  try {
+    Random r = new Random(5678);
+    for (int c = 0; c < 10; c++) {
+      testVectorSerializeRow(c, r, SerializationType.BINARY_SORTABLE);
+    }
+    for (int c = 0; c < 10; c++) {
+      testVectorSerializeRow(c, r, SerializationType.LAZY_BINARY);
+    }
+    for (int c = 0; c < 10; c++) {
+      testVectorSerializeRow(c, r, SerializationType.LAZY_SIMPLE);
+    }
+
+    for (int c = 0; c < 10; c++) {
+      testVectorDeserializeRow(c, r, SerializationType.BINARY_SORTABLE);
+    }
+    for (int c = 0; c < 10; c++) {
+      testVectorDeserializeRow(c, r, SerializationType.LAZY_BINARY);
+    }
+    for (int c = 0; c < 10; c++) {
+      testVectorDeserializeRow(c, r, SerializationType.LAZY_SIMPLE);
+    }
+
+
+  } catch (Throwable e) {
+    e.printStackTrace();
+    throw e;
+  }
+  }
+}
\ No newline at end of file



Mime
View raw message