drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [1/6] drill git commit: DRILL-3232: Promotable writer
Date Tue, 03 Nov 2015 08:44:34 GMT
Repository: drill
Updated Branches:
  refs/heads/master ca5a8476f -> bb69f2202


DRILL-3232: Promotable writer


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/31263e1d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/31263e1d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/31263e1d

Branch: refs/heads/master
Commit: 31263e1d986ce43797e97d91b96c88e5ebf24873
Parents: eeb05fb
Author: Steven Phillips <smp@apache.org>
Authored: Fri Oct 9 12:59:37 2015 -0700
Committer: Steven Phillips <smp@apache.org>
Committed: Mon Nov 2 22:28:32 2015 -0800

----------------------------------------------------------------------
 .../drill/common/expression/IfExpression.java   |  17 ++
 .../codegen/templates/AbstractFieldReader.java  |  10 +-
 .../AbstractPromotableFieldWriter.java          | 142 ++++++++++++++
 .../src/main/codegen/templates/BaseReader.java  |   1 +
 .../src/main/codegen/templates/MapWriters.java  |  10 +-
 .../src/main/codegen/templates/TypeHelper.java  |   4 +-
 .../main/codegen/templates/UnionListWriter.java |  18 +-
 .../src/main/codegen/templates/UnionReader.java |   6 +
 .../src/main/codegen/templates/UnionVector.java |   7 +-
 .../org/apache/drill/exec/ExecConstants.java    |   2 +-
 .../drill/exec/expr/EvaluationVisitor.java      |   4 +-
 .../exec/expr/ExpressionTreeMaterializer.java   |  21 +--
 .../exec/expr/fn/DrillSimpleFuncHolder.java     |  11 +-
 .../drill/exec/expr/fn/impl/UnionFunctions.java |  19 ++
 .../IteratorValidatorBatchIterator.java         |   2 +
 .../drill/exec/record/SimpleVectorWrapper.java  |   7 +
 .../apache/drill/exec/record/TypedFieldId.java  |  18 +-
 .../apache/drill/exec/util/BatchPrinter.java    |  11 +-
 .../vector/complex/AbstractContainerVector.java |   4 +
 .../exec/vector/complex/AbstractMapVector.java  |   2 +-
 .../vector/complex/BaseRepeatedValueVector.java |   5 +
 .../drill/exec/vector/complex/ListVector.java   | 112 +++++++++--
 .../vector/complex/impl/AbstractBaseReader.java |   8 +
 .../vector/complex/impl/PromotableWriter.java   | 188 +++++++++++++++++++
 .../vector/complex/impl/UnionListReader.java    |  29 ++-
 .../org/apache/drill/TestExampleQueries.java    |  17 +-
 .../drill/exec/store/TestOutputMutator.java     |  16 ++
 .../complex/writer/TestComplexTypeReader.java   |   6 +
 .../vector/complex/writer/TestJsonReader.java   |  18 ++
 .../complex/writer/TestPromotableWriter.java    |  66 +++++++
 .../src/test/resources/jsoninput/union/b.json   |   3 +
 31 files changed, 716 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
index c0306ce..ab1e4a2 100644
--- a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
+++ b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
@@ -19,10 +19,13 @@ package org.apache.drill.common.expression;
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 
+import com.google.common.collect.Sets;
 import org.apache.drill.common.expression.visitors.ExprVisitor;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -92,6 +95,20 @@ public class IfExpression extends LogicalExpressionBase {
     // If the return type of one of the "then" expression or "else" expression is nullable, return "if" expression
     // type as nullable
     MajorType majorType = elseExpression.getMajorType();
+    if (majorType.getMinorType() == MinorType.UNION) {
+      Set<MinorType> subtypes = Sets.newHashSet();
+      for (MinorType subtype : majorType.getSubTypeList()) {
+        subtypes.add(subtype);
+      }
+      for (MinorType subtype : ifCondition.expression.getMajorType().getSubTypeList()) {
+        subtypes.add(subtype);
+      }
+      MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL);
+      for (MinorType subtype : subtypes) {
+        builder.addSubType(subtype);
+      }
+      return builder.build();
+    }
     if (majorType.getMode() == DataMode.OPTIONAL) {
       return majorType;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
index 89afd7c..a4d22cf 100644
--- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
+++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
@@ -33,6 +33,10 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
     super();
   }
 
+  public boolean isSet() {
+    return true;
+  }
+
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
           "Character", "DateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
@@ -57,9 +61,9 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
   public void copyAsField(String name, MapWriter writer){
     fail("CopyAsField MapWriter");
   }
-  public void copyAsValue(ListWriter writer){
-    fail("CopyAsValueList");
-  }
+//  public void copyAsValue(ListWriter writer){
+//    fail("CopyAsValueList");
+//  }
   public void copyAsField(String name, ListWriter writer){
     fail("CopyAsFieldList");
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
new file mode 100644
index 0000000..f2f8193
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractPromotableFieldWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * A FieldWriter which delegates calls to another FieldWriter. The delegate FieldWriter can be promoted to a new type
+ * when necessary. Classes that extend this class are responsible for handling promotion.
+ *
+ * This class is generated using freemarker and the ${.template_name} template.
+ *
+ */
+@SuppressWarnings("unused")
+abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
+  AbstractPromotableFieldWriter(FieldWriter parent) {
+    super(parent);
+  }
+
+  /**
+   * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
+   * @param type
+   * @return
+   */
+  abstract protected FieldWriter getWriter(MinorType type);
+
+  /**
+   * Return the current FieldWriter
+   * @return
+   */
+  abstract protected FieldWriter getWriter();
+
+  @Override
+  public void start() {
+    getWriter(MinorType.MAP).start();
+  }
+
+  @Override
+  public void end() {
+    getWriter(MinorType.MAP).end();
+  }
+
+  @Override
+  public void startList() {
+    getWriter(MinorType.LIST).startList();
+  }
+
+  @Override
+  public void endList() {
+    getWriter(MinorType.LIST).endList();
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#if !minor.class?starts_with("Decimal") >
+  @Override
+  public void write(${name}Holder holder) {
+    getWriter(MinorType.${name?upper_case}).write(holder);
+  }
+
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+  }
+
+  </#if>
+  </#list></#list>
+
+  public void writeNull() {
+  }
+
+  @Override
+  public MapWriter map() {
+    return getWriter(MinorType.LIST).map();
+  }
+
+  @Override
+  public ListWriter list() {
+    return getWriter(MinorType.LIST).list();
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    return getWriter(MinorType.MAP).map(name);
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    return getWriter(MinorType.MAP).list(name);
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if !minor.class?starts_with("Decimal") >
+
+  @Override
+  public ${capName}Writer ${lowerName}(String name) {
+    return getWriter(MinorType.MAP).${lowerName}(name);
+  }
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    return getWriter(MinorType.LIST).${lowerName}();
+  }
+
+  </#if>
+  </#list></#list>
+
+  public void copyReader(FieldReader reader) {
+    getWriter().copyReader(reader);
+  }
+
+  public void copyReaderToField(String name, FieldReader reader) {
+    getWriter().copyReaderToField(name, reader);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/BaseReader.java b/exec/java-exec/src/main/codegen/templates/BaseReader.java
index 4fce409..78f32f4 100644
--- a/exec/java-exec/src/main/codegen/templates/BaseReader.java
+++ b/exec/java-exec/src/main/codegen/templates/BaseReader.java
@@ -36,6 +36,7 @@ public interface BaseReader extends Positionable{
   void read(UnionHolder holder);
   void read(int index, UnionHolder holder);
   void copyAsValue(UnionWriter writer);
+  boolean isSet();
 
   public interface MapReader extends BaseReader, Iterable<String>{
     FieldReader reader(String name);

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java
index 42cacab..e6becf2 100644
--- a/exec/java-exec/src/main/codegen/templates/MapWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java
@@ -85,8 +85,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
         MapVector vector=container.addOrGet(name,MapVector.TYPE,MapVector.class);
         writer=new SingleMapWriter(vector,this);
       } else {
-        UnionVector vector = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
-        writer = new UnionWriter(vector);
+        MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
+        writer = new PromotableWriter(vector, container);
       }
       if(vectorCount != container.size()) {
         writer.allocate();
@@ -127,7 +127,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
       if (!unionEnabled){
         writer = new SingleListWriter(name,container,this);
       } else{
-        writer = new UnionWriter(container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class));
+        writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
       }
       if (container.size() > vectorCount) {
         writer.allocate();
@@ -215,8 +215,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     if(writer == null) {
       ValueVector vector;
       if (unionEnabled){
-        UnionVector v = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
-        writer = new UnionWriter(v);
+        ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
+        writer = new PromotableWriter(v, container);
         vector = v;
       } else {
         ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index d9810c7..b8178cb 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -206,6 +206,7 @@ public class TypeHelper {
     case MAP:
       switch (mode) {
       case REQUIRED:
+      case OPTIONAL:
         return SingleMapWriter.class;
       case REPEATED:
         return RepeatedMapWriter.class;
@@ -213,7 +214,8 @@ public class TypeHelper {
     case LIST:
       switch (mode) {
       case REQUIRED:
-        return SingleListWriter.class;
+      case OPTIONAL:
+        return UnionListWriter.class;
       case REPEATED:
         return RepeatedListWriter.class;
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
index fd7256b..35c0bc1 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
@@ -34,9 +34,9 @@ package org.apache.drill.exec.vector.complex.impl;
 public class UnionListWriter extends AbstractFieldWriter {
 
   ListVector vector;
-  UnionVector data;
+//  UnionVector data;
   UInt4Vector offsets;
-  private UnionWriter writer;
+  private PromotableWriter writer;
   private boolean inMap = false;
   private String mapName;
   private int lastIndex = 0;
@@ -44,11 +44,15 @@ public class UnionListWriter extends AbstractFieldWriter {
   public UnionListWriter(ListVector vector) {
     super(null);
     this.vector = vector;
-    this.data = (UnionVector) vector.getDataVector();
-    this.writer = new UnionWriter(data);
+//    this.data = (UnionVector) vector.getDataVector();
+    this.writer = new PromotableWriter(vector.getDataVector(), vector);
     this.offsets = vector.getOffsetVector();
   }
 
+  public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
+    this(vector);
+  }
+
   @Override
   public void allocate() {
     vector.allocateNew();
@@ -115,7 +119,7 @@ public class UnionListWriter extends AbstractFieldWriter {
   public ListWriter list(String name) {
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
-    data.getMutator().setType(nextOffset, MinorType.MAP);
+//    data.getMutator().setType(nextOffset, MinorType.MAP);
     writer.setPosition(nextOffset);
     ListWriter listWriter = writer.list(name);
     return listWriter;
@@ -142,7 +146,7 @@ public class UnionListWriter extends AbstractFieldWriter {
     assert inMap;
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
-    data.getMutator().setType(nextOffset, MinorType.MAP);
+//    data.getMutator().setType(nextOffset, MinorType.MAP);
     offsets.getMutator().setSafe(idx() + 1, nextOffset);
     writer.setPosition(nextOffset);
   }
@@ -167,7 +171,7 @@ public class UnionListWriter extends AbstractFieldWriter {
     if (inMap) {
       final int nextOffset = offsets.getAccessor().get(idx() + 1);
       vector.getMutator().setNotNull(idx());
-      data.getMutator().setType(nextOffset, MinorType.MAP);
+//      data.getMutator().setType(nextOffset, MinorType.MAP);
       writer.setPosition(nextOffset);
       ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(mapName);
       ${uncappedName}Writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java
index 38d4247..4943ea0 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionReader.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java
@@ -157,6 +157,12 @@ public class UnionReader extends AbstractFieldReader {
   </#list></#list>
 
   @Override
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer);
+    copier.write();
+  }
+
+  @Override
   public void setPosition(int index) {
     super.setPosition(index);
     for (BaseReader reader : readers) {

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java
index bbd09c2..3e7b3bf 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionVector.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java
@@ -75,7 +75,7 @@ public class UnionVector implements ValueVector {
     internalMapWriter = new SingleMapWriter(internalMap, null, true, true);
     this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class);
     this.field.addChild(internalMap.getField().clone());
-    this.majorType = Types.optional(MinorType.UNION);
+    this.majorType = field.getType();
     this.callBack = callBack;
   }
 
@@ -243,6 +243,11 @@ public class UnionVector implements ValueVector {
     copyFrom(inIndex, outIndex, from);
   }
 
+  public void addVector(ValueVector v) {
+    internalMap.putChild(v.getField().getType().getMinorType().name().toLowerCase(), v);
+    addSubType(v.getField().getType().getMinorType());
+  }
+
   private class TransferImpl implements TransferPair {
 
     UnionVector to;

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 45d0304..f85a776 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -160,7 +160,7 @@ public interface ExecConstants {
   public static String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double";
   public static OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false);
 
-  public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", true);
+  public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false);
 
   // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare
   // in core which is not right. Move this option and above two mongo plugin related options once we have the feature.

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 19aa3c9..424174e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -21,6 +21,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
+import com.google.common.collect.Maps;
+import com.sun.codemodel.JOp;
 import org.apache.drill.common.expression.BooleanOperator;
 import org.apache.drill.common.expression.CastExpression;
 import org.apache.drill.common.expression.ConvertExpression;
@@ -387,7 +389,7 @@ public class EvaluationVisitor {
           if (seg.isArray()) {
             // stop once we get to the last segment and the final type is neither complex nor repeated (map, list, repeated list).
             // In case of non-complex and non-repeated type, we return Holder, in stead of FieldReader.
-            if (seg.isLastPath() && !complex && !repeated) {
+            if (seg.isLastPath() && !complex && !repeated && !e.getTypedFieldId().isListVector()) {
               break;
             }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 621fb0e..827623b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -382,6 +382,9 @@ public class ExpressionTreeMaterializer {
     }
 
     private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) {
+      if (type == MinorType.UNION) {
+        return null;
+      }
       String castFuncName = String.format("as%s", type.toString());
       List<LogicalExpression> args = Lists.newArrayList();
       args.add(arg);
@@ -406,6 +409,7 @@ public class ExpressionTreeMaterializer {
 
       MinorType thenType = conditions.expression.getMajorType().getMinorType();
       MinorType elseType = newElseExpr.getMajorType().getMinorType();
+      boolean hasUnion = thenType == MinorType.UNION || elseType == MinorType.UNION;
       if (unionTypeEnabled) {
         if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) {
 
@@ -426,18 +430,9 @@ public class ExpressionTreeMaterializer {
             builder.addSubType(elseType);
           }
           MajorType unionType = builder.build();
-//          if (leastRestrictive != thenType) {
-            // Implicitly cast the then expression
-            conditions = new IfExpression.IfCondition(newCondition,
-                    addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector));
-//          } if (leastRestrictive != elseType) {
-            // Implicitly cast the else expression
-            newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector);
-//          } else {
-//            conditions = new IfExpression.IfCondition(newCondition,
-//                    addCastExpression(conditions.expression, Types.optional(MinorType.UNION), functionLookupContext, errorCollector));
-//            newElseExpr = addCastExpression(newElseExpr, Types.optional(MinorType.UNION), functionLookupContext, errorCollector);
-//          }
+          conditions = new IfExpression.IfCondition(newCondition,
+                  addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector));
+          newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector);
         }
 
       } else {
@@ -492,7 +487,7 @@ public class ExpressionTreeMaterializer {
         }
       }
 
-      if (!unionTypeEnabled) {
+      if (!hasUnion) {
         // If the type of the IF expression is nullable, apply a convertToNullable*Holder function for "THEN"/"ELSE"
         // expressions whose type is not nullable.
         if (IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).build().getMajorType().getMode()

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
index 859f270..78e4c62 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.expr.fn;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.sun.codemodel.JOp;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -97,10 +98,16 @@ public class DrillSimpleFuncHolder extends DrillFuncHolder {
       JExpression e = null;
       for (HoldingContainer v : inputVariables) {
         if (v.isOptional()) {
+          JExpression isNullExpr;
+          if (v.isReader()) {
+           isNullExpr = JOp.cond(v.getHolder().invoke("isSet"), JExpr.lit(1), JExpr.lit(0));
+          } else {
+            isNullExpr = v.getIsSet();
+          }
           if (e == null) {
-            e = v.getIsSet();
+            e = isNullExpr;
           } else {
-            e = e.mul(v.getIsSet());
+            e = e.mul(isNullExpr);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
index a4b3cc4..34b14fe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
@@ -24,6 +24,7 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableUInt1Holder;
 import org.apache.drill.exec.expr.holders.UnionHolder;
 import org.apache.drill.exec.expr.holders.IntHolder;
 import org.apache.drill.exec.expr.holders.VarCharHolder;
@@ -114,4 +115,22 @@ public class UnionFunctions {
       out.isSet = in.isSet() ? 1 : 0;
     }
   }
+
+  @SuppressWarnings("unused")
+  @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  public static class CastUnionList implements DrillSimpleFunc {
+
+    @Param UnionHolder in;
+    @Output UnionHolder out;
+
+    public void setup() {}
+
+    public void eval() {
+      if (in.isSet == 1) {
+        out.reader = in.reader;
+      } else {
+        out.isSet = 0;
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
index efd155e..1687c3d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
@@ -30,6 +30,7 @@ import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.record.WritableBatch;
 import org.apache.drill.exec.record.selection.SelectionVector2;
 import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.util.BatchPrinter;
 import org.apache.drill.exec.vector.VectorValidator;
 
 public class IteratorValidatorBatchIterator implements CloseableRecordBatch {
@@ -133,6 +134,7 @@ public class IteratorValidatorBatchIterator implements CloseableRecordBatch {
         throw new IllegalStateException (String.format("Incoming batch of %s has size %d, which is beyond the limit of %d",  incoming.getClass().getName(), incoming.getRecordCount(), MAX_BATCH_SIZE));
       }
 
+//      BatchPrinter.printBatch(incoming);
       if (VALIDATE_VECTORS) {
         VectorValidator.validate(incoming);
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
index 59246ef..2a8e91f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.AbstractContainerVector;
 import org.apache.drill.exec.vector.complex.AbstractMapVector;
+import org.apache.drill.exec.vector.complex.ListVector;
 import org.apache.drill.exec.vector.complex.MapVector;
 import org.apache.drill.exec.vector.complex.impl.UnionVector;
 
@@ -125,6 +126,12 @@ public class SimpleVectorWrapper<T extends ValueVector> implements VectorWrapper
       builder.finalType(majorType);
       builder.intermediateType(majorType);
       return builder.build();
+    } else if (v instanceof ListVector) {
+      ListVector list = (ListVector) v;
+      TypedFieldId.Builder builder = TypedFieldId.newBuilder();
+      builder.intermediateType(v.getField().getType());
+      builder.addId(id);
+      return list.getFieldIdIfMatches(builder, true, expectedPath.getRootSegment().getChild());
     } else
     if (v instanceof AbstractContainerVector) {
       // we're looking for a multi path.

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
index db8bfa8..931f981 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
@@ -34,6 +34,7 @@ public class TypedFieldId {
   final MajorType intermediateType;
   final int[] fieldIds;
   final boolean isHyperReader;
+  final boolean isListVector;
   final PathSegment remainder;
 
   public TypedFieldId(MajorType type, int... fieldIds) {
@@ -49,12 +50,17 @@ public class TypedFieldId {
   }
 
   public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, PathSegment remainder, int... fieldIds) {
+    this(intermediateType, secondaryFinal, finalType, isHyper, false, remainder, fieldIds);
+  }
+
+  public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, boolean isListVector, PathSegment remainder, int... fieldIds) {
     super();
     this.intermediateType = intermediateType;
     this.finalType = finalType;
     this.secondaryFinal = secondaryFinal;
     this.fieldIds = fieldIds;
     this.isHyperReader = isHyper;
+    this.isListVector = isListVector;
     this.remainder = remainder;
   }
 
@@ -90,6 +96,10 @@ public class TypedFieldId {
     return isHyperReader;
   }
 
+  public boolean isListVector() {
+    return isListVector;
+  }
+
   public MajorType getIntermediateType() {
     return intermediateType;
   }
@@ -122,6 +132,7 @@ public class TypedFieldId {
     PathSegment remainder;
     boolean hyperReader = false;
     boolean withIndex = false;
+    boolean isListVector = false;
 
     public Builder addId(int id) {
       ids.add(id);
@@ -143,6 +154,11 @@ public class TypedFieldId {
       return this;
     }
 
+    public Builder listVector() {
+      this.isListVector = true;
+      return this;
+    }
+
     public Builder finalType(MajorType finalType) {
       this.finalType = finalType;
       return this;
@@ -179,7 +195,7 @@ public class TypedFieldId {
       // TODO: there is a bug here with some things.
       //if(intermediateType != finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.OPTIONAL).build();
 
-      return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, remainder, ids.toArray());
+      return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, isListVector, remainder, ids.toArray());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
index 6021cd2..55173f9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
@@ -75,13 +75,16 @@ public class BatchPrinter {
       }
       for (ValueVector vv : vectors) {
         Object o = vv.getAccessor().getObject(row);
+        String value;
+        if (o == null) {
+          value = "null";
+        } else
         if (o instanceof byte[]) {
-          String value = new String((byte[]) o);
-          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+          value = new String((byte[]) o);
         } else {
-          String value = o.toString();
-          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+          value = o.toString();
         }
+        System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
       }
       System.out.printf("|\n");
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index d14dca6..17b42a1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -22,6 +22,7 @@ import java.util.Collection;
 import java.util.Collections;
 
 import javax.annotation.Nullable;
+import javax.validation.constraints.DecimalMin.List;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
@@ -155,6 +156,9 @@ public abstract class AbstractContainerVector implements ValueVector {
       // we're looking for a multi path.
       AbstractContainerVector c = (AbstractContainerVector) v;
       return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+    } else if(v instanceof ListVector) {
+      ListVector list = (ListVector) v;
+      return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
     } else {
       if (seg.isNamed()) {
         if(addToBreadCrumb) {

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index 2c93c31..1706f5c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -173,7 +173,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
    */
-  protected void putChild(String name, ValueVector vector) {
+  public void putChild(String name, ValueVector vector) {
     putVector(name, vector);
     field.addChild(vector.getField());
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
index 19b5ab4..29f6cda 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -202,6 +202,11 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     return new AddOrGetResult<>((T)vector, created);
   }
 
+  protected void replaceDataVector(ValueVector v) {
+    vector.clear();
+    vector = v;
+  }
+
   public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
 
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index ccd6239..a44efe3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -21,6 +21,8 @@ package org.apache.drill.exec.vector.complex;
 import com.google.common.collect.ObjectArrays;
 import io.netty.buffer.DrillBuf;
 import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.memory.BufferAllocator;
@@ -28,8 +30,10 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
 import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.util.CallBack;
 import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.vector.AddOrGetResult;
 import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.UInt1Vector;
 import org.apache.drill.exec.vector.UInt4Vector;
@@ -46,20 +50,22 @@ import java.util.List;
 
 public class ListVector extends BaseRepeatedValueVector {
 
-  UInt4Vector offsets;
-  protected final UInt1Vector bits;
-  Mutator mutator = new Mutator();
-  Accessor accessor = new Accessor();
-  UnionListWriter writer;
-  UnionListReader reader;
+  private UInt4Vector offsets;
+  private final UInt1Vector bits;
+  private Mutator mutator = new Mutator();
+  private Accessor accessor = new Accessor();
+  private UnionListWriter writer;
+  private UnionListReader reader;
+  private CallBack callBack;
 
   public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    super(field, allocator, new UnionVector(field, allocator, callBack));
+    super(field, allocator);
     this.bits = new UInt1Vector(MaterializedField.create("$bits$", Types.required(MinorType.UINT1)), allocator);
     offsets = getOffsetVector();
     this.field.addChild(getDataVector().getField());
     this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
+    this.callBack = callBack;
   }
 
   public UnionListWriter getWriter() {
@@ -87,8 +93,8 @@ public class ListVector extends BaseRepeatedValueVector {
   }
 
   @Override
-  public UnionVector getDataVector() {
-    return (UnionVector) vector;
+  public ValueVector getDataVector() {
+    return vector;
   }
 
   @Override
@@ -107,10 +113,12 @@ public class ListVector extends BaseRepeatedValueVector {
 
     public TransferImpl(MaterializedField field) {
       to = new ListVector(field, allocator, null);
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
     }
 
     public TransferImpl(ListVector to) {
       this.to = to;
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
     }
 
     @Override
@@ -182,6 +190,11 @@ public class ListVector extends BaseRepeatedValueVector {
             .addChild(bits.getMetadata())
             .addChild(vector.getMetadata());
   }
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+    reader = new UnionListReader(this);
+    return result;
+  }
 
   @Override
   public int getBufferSize() {
@@ -232,20 +245,97 @@ public class ListVector extends BaseRepeatedValueVector {
     vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
   }
 
+  public UnionVector promoteToUnion() {
+    MaterializedField newField = MaterializedField.create(getField().getPath(), Types.optional(MinorType.UNION));
+    UnionVector vector = new UnionVector(newField, allocator, null);
+    replaceDataVector(vector);
+    reader = new UnionListReader(this);
+    return vector;
+  }
+
+  public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+    if (seg == null) {
+      if (addToBreadCrumb) {
+        builder.intermediateType(this.getField().getType());
+      }
+      return builder.finalType(this.getField().getType()).build();
+    }
+
+    if (seg.isArray()) {
+      if (seg.isLastPath()) {
+        builder //
+                .withIndex() //
+                .listVector()
+                .finalType(getDataVector().getField().getType());
+
+        // remainder starts with the 1st array segment in SchemaPath.
+        // only set remainder when it's the only array segment.
+        if (addToBreadCrumb) {
+          addToBreadCrumb = false;
+          builder.remainder(seg);
+        }
+        return builder.build();
+      } else {
+        if (addToBreadCrumb) {
+          addToBreadCrumb = false;
+          builder.remainder(seg);
+        }
+      }
+    } else {
+      return null;
+    }
+
+    ValueVector v = getDataVector();
+    if (v instanceof AbstractContainerVector) {
+      // we're looking for a multi path.
+      AbstractContainerVector c = (AbstractContainerVector) v;
+      return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+    } else if (v instanceof ListVector) {
+      ListVector list = (ListVector) v;
+      return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+    } else {
+      if (seg.isNamed()) {
+        if(addToBreadCrumb) {
+          builder.intermediateType(v.getField().getType());
+        }
+        builder.finalType(v.getField().getType());
+      } else {
+        builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
+      }
+
+      if (seg.isLastPath()) {
+        return builder.build();
+      } else {
+        PathSegment child = seg.getChild();
+        if (child.isLastPath() && child.isArray()) {
+          if (addToBreadCrumb) {
+            builder.remainder(child);
+          }
+          builder.withIndex();
+          builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
+          return builder.build();
+        } else {
+//          logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
+          return null;
+        }
+      }
+    }
+  }
+
   private int lastSet;
 
   public class Accessor extends BaseRepeatedAccessor {
 
     @Override
     public Object getObject(int index) {
-      if (bits.getAccessor().isNull(index)) {
+      if (isNull(index)) {
         return null;
       }
       final List<Object> vals = new JsonStringArrayList<>();
       final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
       final int start = offsetsAccessor.get(index);
       final int end = offsetsAccessor.get(index + 1);
-      final UnionVector.Accessor valuesAccessor = getDataVector().getAccessor();
+      final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
       for(int i = start; i < end; i++) {
         vals.add(valuesAccessor.getObject(i));
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
index fea326c..7624b1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
@@ -24,6 +24,8 @@ import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.holders.UnionHolder;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
 
 
 abstract class AbstractBaseReader implements FieldReader{
@@ -88,4 +90,10 @@ abstract class AbstractBaseReader implements FieldReader{
   public void copyAsValue(UnionWriter writer) {
     throw new IllegalStateException("The current reader doesn't support reading union type");
   }
+
+  @Override
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier copier = new ComplexCopier(this, (FieldWriter)writer);
+    copier.write();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
new file mode 100644
index 0000000..d3459c3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.impl;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.ZeroVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
+import org.apache.drill.exec.vector.complex.ListVector;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+import java.lang.reflect.Constructor;
+
+/**
+ * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
+ * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
+ * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector
+ * in the parent vector, which can be either an AbstractMapVector or a ListVector.
+ */
+public class PromotableWriter extends AbstractPromotableFieldWriter {
+
+  private final AbstractMapVector parentContainer;
+  private final ListVector listVector;
+  private int position;
+
+  private enum State {
+    UNTYPED, SINGLE, UNION
+  }
+
+  private MinorType type;
+  private ValueVector vector;
+  private UnionVector unionVector;
+  private State state;
+  private FieldWriter writer;
+
+  public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
+    super(null);
+    this.parentContainer = parentContainer;
+    this.listVector = null;
+    init(v);
+  }
+
+  public PromotableWriter(ValueVector v, ListVector listVector) {
+    super(null);
+    this.listVector = listVector;
+    this.parentContainer = null;
+    init(v);
+  }
+
+  private void init(ValueVector v) {
+    if (v instanceof UnionVector) {
+      state = State.UNION;
+      unionVector = (UnionVector) v;
+      writer = new UnionWriter(unionVector);
+    } else if (v instanceof ZeroVector) {
+      state = State.UNTYPED;
+    } else {
+      setWriter(v);
+    }
+  }
+
+  private void setWriter(ValueVector v) {
+    state = State.SINGLE;
+    vector = v;
+    type = v.getField().getType().getMinorType();
+    Class writerClass = TypeHelper.getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
+    if (writerClass.equals(SingleListWriter.class)) {
+      writerClass = UnionListWriter.class;
+    }
+    Class vectorClass = TypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField().getDataMode());
+    try {
+      Constructor constructor = null;
+      for (Constructor c : writerClass.getConstructors()) {
+        if (c.getParameterTypes().length == 4) {
+          constructor = c;
+        }
+      }
+      if (constructor == null) {
+        constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
+        writer = (FieldWriter) constructor.newInstance(vector, null);
+      } else {
+        writer = (FieldWriter) constructor.newInstance(vector, null, true, false);
+      }
+    } catch (ReflectiveOperationException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    FieldWriter w = getWriter();
+    if (w == null) {
+      position = index;
+    } else {
+      w.setPosition(index);
+    }
+  }
+
+  protected FieldWriter getWriter(MinorType type) {
+    if (state == State.UNION) {
+      return writer;
+    }
+    if (state == State.UNTYPED) {
+      if (type == null) {
+        return null;
+      }
+      ValueVector v = listVector.addOrGetVector(new VectorDescriptor(Types.optional(type))).getVector();
+      v.allocateNew();
+      setWriter(v);
+      writer.setPosition(position);
+    }
+    if (type != this.type) {
+      return promoteToUnion();
+    }
+    return writer;
+  }
+
+  protected FieldWriter getWriter() {
+    return getWriter(type);
+  }
+
+  private FieldWriter promoteToUnion() {
+    String name = vector.getField().getLastName();
+    TransferPair tp = vector.getTransferPair(new FieldReference(vector.getField().getType().getMinorType().name().toLowerCase()));
+    tp.transfer();
+    if (parentContainer != null) {
+      unionVector = parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
+    } else if (listVector != null) {
+      unionVector = listVector.promoteToUnion();
+    }
+    unionVector.addVector(tp.getTo());
+    writer = new UnionWriter(unionVector);
+    writer.setPosition(idx());
+    for (int i = 0; i < idx(); i++) {
+      unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+    }
+    vector = null;
+    state = State.UNION;
+    return writer;
+  }
+
+  @Override
+  public void allocate() {
+    getWriter().allocate();
+  }
+
+  @Override
+  public void clear() {
+    getWriter().clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return getWriter().getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return getWriter().getValueCapacity();
+  }
+
+  @Override
+  public void close() throws Exception {
+    getWriter().close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
index fef3325..db2d8e4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
@@ -19,23 +19,28 @@
 package org.apache.drill.exec.vector.complex.impl;
 
 import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.holders.UnionHolder;
 import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.ListVector;
 import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
 
 public class UnionListReader extends AbstractFieldReader {
 
   private ListVector vector;
-  private UnionVector data;
+  private ValueVector data;
   private UInt4Vector offsets;
-  private UnionReader reader;
 
   public UnionListReader(ListVector vector) {
     this.vector = vector;
     this.data = vector.getDataVector();
     this.offsets = vector.getOffsetVector();
-    this.reader = (UnionReader) data.getReader();
   }
 
   @Override
@@ -43,9 +48,10 @@ public class UnionListReader extends AbstractFieldReader {
     return true;
   }
 
-  @Override
+  MajorType type = Types.optional(MinorType.LIST);
+
   public MajorType getType() {
-    return reader.getType();
+    return type;
   }
 
   private int currentOffset;
@@ -60,7 +66,7 @@ public class UnionListReader extends AbstractFieldReader {
 
   @Override
   public FieldReader reader() {
-    return reader;
+    return data.getReader();
   }
 
   @Override
@@ -74,17 +80,22 @@ public class UnionListReader extends AbstractFieldReader {
     for (int i = -1; i < index; i++) {
       next();
     }
-    holder.reader = reader;
-    holder.isSet = reader.isSet() ? 1 : 0;
+    holder.reader = data.getReader();
+    holder.isSet = data.getReader().isSet() ? 1 : 0;
   }
 
   @Override
   public boolean next() {
     if (currentOffset + 1 < maxOffset) {
-      reader.setPosition(++currentOffset);
+      data.getReader().setPosition(++currentOffset);
       return true;
     } else {
       return false;
     }
   }
+
+  public void copyAsValue(ListWriter writer) {
+    ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer);
+    copier.write();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index e952bff..05bd138 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -26,20 +26,21 @@ import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.util.FileUtils;
 import org.apache.drill.common.util.TestTools;
 import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.After;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class TestExampleQueries extends BaseTestQuery {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
 
-  @Test
-  public void q() throws Exception {
-    testNoResult("use dfs.tmp");
-//    test("select 1 + cast(a as bigint) from t");
-//    test("select cast(1 as bigint) + case typeOf(a) when type('bigint') then asBigInt(a) when type('varchar') then cast(asVarChar(a) as bigint) end from t");
-//    test("select a from t where typeOf(a) = type('bigint')");
-    test("select a_b, typeOf(a_b) type from (select a + b as a_b from t3)");
-//    test("select sum(a) from t");
+  @After
+  public void reset() throws Exception {
+    String[] options = new String[] { ExecConstants.SLICE_TARGET, PlannerSettings.HASHJOIN.getOptionName(), PlannerSettings.HASHAGG.getOptionName(),
+            PlannerSettings.STREAMAGG.getOptionName(), PlannerSettings.MERGEJOIN.getOptionName(), PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR.getOptionName() };
+    for (String option : options) {
+      testNoResult(String.format("reset `%s`", option));
+    }
   }
 
   @Test // see DRILL-2328

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
index 51b909b..0c5ff49 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
@@ -17,12 +17,14 @@
  */
 package org.apache.drill.exec.store;
 
+import com.google.common.collect.Lists;
 import io.netty.buffer.DrillBuf;
 
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.BufferAllocator;
@@ -60,6 +62,20 @@ public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper<
     fieldVectorMap.put(vector.getField(), vector);
   }
 
+  private void replace(ValueVector newVector, SchemaPath schemaPath) {
+    List<ValueVector> vectors = Lists.newArrayList();
+    for (VectorWrapper w : container) {
+      ValueVector vector = w.getValueVector();
+      if (vector.getField().getPath().equals(schemaPath)) {
+        vectors.add(newVector);
+      } else {
+        vectors.add(w.getValueVector());
+      }
+      container.remove(vector);
+    }
+    container.addCollection(vectors);
+  }
+
   public Iterator<VectorWrapper<?>> iterator() {
     return container.iterator();
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 521a41d..7f46841 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -19,11 +19,17 @@
 package org.apache.drill.exec.vector.complex.writer;
 
 import org.apache.drill.BaseTestQuery;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestComplexTypeReader extends BaseTestQuery{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class);
 
+  @BeforeClass
+  public static void init() throws Exception {
+    testNoResult("alter session set `exec.enable_union_type` = true");
+  }
+
   @Test
   // Repeated map (map) -> json.
   public void testX() throws Exception{

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 0ae8945..d549a19 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -482,4 +482,22 @@ public class TestJsonReader extends BaseTestQuery {
     }
   }
 
+  @Test
+  public void testUnionExpressionMaterialization() throws Exception {
+    String query = "select a + b c from cp.`jsoninput/union/b.json`";
+    try {
+      testBuilder()
+              .sqlQuery(query)
+              .ordered()
+              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
+              .baselineColumns("c")
+              .baselineValues(3L)
+              .baselineValues(7.0)
+              .baselineValues(11.0)
+              .go();
+    } finally {
+      testNoResult("alter session set `exec.enable_union_type` = false");
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java
new file mode 100644
index 0000000..60a2268
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.writer;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.RootAllocatorFactory;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.store.TestOutputMutator;
+import org.apache.drill.exec.util.BatchPrinter;
+import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.junit.Test;
+
+public class TestPromotableWriter {
+
+  @Test
+  public void list() throws Exception {
+    BufferAllocator allocator = RootAllocatorFactory.newRoot(DrillConfig.create());
+    TestOutputMutator output = new TestOutputMutator(allocator);
+    ComplexWriter rootWriter = new VectorContainerWriter(output, true);
+    MapWriter writer = rootWriter.rootAsMap();
+
+
+    rootWriter.setPosition(0);
+    {
+      writer.map("map").bigInt("a").writeBigInt(1);
+    }
+    rootWriter.setPosition(1);
+    {
+      writer.map("map").float4("a").writeFloat4(2.0f);
+    }
+    rootWriter.setPosition(2);
+    {
+      writer.map("map").list("a").startList();
+      writer.map("map").list("a").endList();
+    }
+    rootWriter.setPosition(3);
+    {
+      writer.map("map").list("a").startList();
+      writer.map("map").list("a").bigInt().writeBigInt(3);
+      writer.map("map").list("a").float4().writeFloat4(4);
+      writer.map("map").list("a").endList();
+    }
+    rootWriter.setValueCount(4);
+    BatchPrinter.printBatch(output.getContainer());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/31263e1d/exec/java-exec/src/test/resources/jsoninput/union/b.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/jsoninput/union/b.json b/exec/java-exec/src/test/resources/jsoninput/union/b.json
new file mode 100644
index 0000000..28a6c5c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/jsoninput/union/b.json
@@ -0,0 +1,3 @@
+{ a :  1, b : "2" }
+{ a : 3.0, b : "4"}
+{ a : 5.0, b : 6 }


Mime
View raw message