drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [6/6] drill git commit: DRILL-3229: Miscellaneous Union-type fixes
Date Tue, 03 Nov 2015 08:44:39 GMT
DRILL-3229: Miscellaneous Union-type fixes

closes #207
closes #180


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bb69f220
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bb69f220
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bb69f220

Branch: refs/heads/master
Commit: bb69f2202ed6115b39bd8681e59c6ff6091e9b9e
Parents: 31263e1
Author: Steven Phillips <smp@apache.org>
Authored: Wed Oct 21 17:00:44 2015 -0700
Committer: Steven Phillips <smp@apache.org>
Committed: Tue Nov 3 00:40:32 2015 -0800

----------------------------------------------------------------------
 .../drill/common/expression/IfExpression.java   |  18 ++-
 .../org/apache/drill/common/types/Types.java    |   9 +-
 .../codegen/templates/AbstractFieldReader.java  |   8 +-
 .../main/codegen/templates/ComplexCopier.java   |  23 ++-
 .../templates/EventBasedRecordWriter.java       |  10 +-
 .../codegen/templates/HolderReaderImpl.java     |   1 +
 .../src/main/codegen/templates/MapWriters.java  |  18 +--
 .../src/main/codegen/templates/NullReader.java  |   2 +-
 .../main/codegen/templates/UnionFunctions.java  |  34 ++--
 .../main/codegen/templates/UnionListWriter.java |  36 ++---
 .../src/main/codegen/templates/UnionReader.java |  13 +-
 .../src/main/codegen/templates/UnionVector.java | 107 +++++++------
 .../src/main/codegen/templates/UnionWriter.java |  60 ++++----
 .../apache/drill/exec/expr/CloneVisitor.java    |   3 +
 .../drill/exec/expr/EvaluationVisitor.java      |   8 +-
 .../exec/expr/ExpressionTreeMaterializer.java   | 154 +++++++++++++++----
 .../drill/exec/expr/fn/ExceptionFunction.java   |  53 +++++++
 .../drill/exec/expr/fn/impl/UnionFunctions.java | 145 +++++++++++------
 .../drill/exec/physical/impl/ImplCreator.java   |   1 +
 .../physical/impl/aggregate/HashAggBatch.java   |   2 +-
 .../impl/aggregate/StreamingAggBatch.java       |   2 +-
 .../physical/impl/filter/FilterRecordBatch.java |   1 +
 .../impl/flatten/FlattenRecordBatch.java        |   4 +-
 .../IteratorValidatorBatchIterator.java         |   1 -
 .../drill/exec/record/SimpleVectorWrapper.java  |  10 +-
 .../exec/vector/accessor/UnionSqlAccessor.java  |   2 +-
 .../vector/complex/AbstractContainerVector.java |  78 +---------
 .../exec/vector/complex/AbstractMapVector.java  |   2 +-
 .../drill/exec/vector/complex/FieldIdUtil.java  | 124 +++++++++++++++
 .../drill/exec/vector/complex/ListVector.java   |  91 +++--------
 .../exec/vector/complex/fn/JsonWriter.java      |   2 +
 .../vector/complex/impl/AbstractBaseReader.java |   3 +-
 .../vector/complex/impl/ComplexWriterImpl.java  |   4 +-
 .../vector/complex/impl/PromotableWriter.java   |   5 +-
 .../vector/complex/impl/UnionListReader.java    |   3 +-
 .../complex/impl/VectorContainerWriter.java     |   2 +-
 .../org/apache/drill/TestExampleQueries.java    |   9 --
 .../physical/impl/writer/TestParquetWriter.java |   1 +
 .../vector/complex/writer/TestJsonReader.java   |  77 +++++++++-
 39 files changed, 696 insertions(+), 430 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
index ab1e4a2..e85caa0 100644
--- a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
+++ b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
@@ -37,11 +37,13 @@ public class IfExpression extends LogicalExpressionBase {
 
   public final IfCondition ifCondition;
   public final LogicalExpression elseExpression;
+  public final MajorType outputType;
 
-  private IfExpression(ExpressionPosition pos, IfCondition conditions, LogicalExpression elseExpression) {
+  private IfExpression(ExpressionPosition pos, IfCondition conditions, LogicalExpression elseExpression, MajorType outputType) {
     super(pos);
     this.ifCondition = conditions;
     this.elseExpression = elseExpression;
+    this.outputType = outputType;
   }
 
   public static class IfCondition{
@@ -66,6 +68,7 @@ public class IfExpression extends LogicalExpressionBase {
     IfCondition conditions;
     private LogicalExpression elseExpression;
     private ExpressionPosition pos = ExpressionPosition.UNKNOWN;
+    private MajorType outputType;
 
     public Builder setPosition(ExpressionPosition pos) {
       this.pos = pos;
@@ -82,18 +85,25 @@ public class IfExpression extends LogicalExpressionBase {
       return this;
     }
 
+    public Builder setOutputType(MajorType outputType) {
+      this.outputType = outputType;
+      return this;
+    }
+
     public IfExpression build(){
       Preconditions.checkNotNull(pos);
       Preconditions.checkNotNull(conditions);
-      return new IfExpression(pos, conditions, elseExpression);
+      return new IfExpression(pos, conditions, elseExpression, outputType);
     }
 
   }
 
   @Override
   public MajorType getMajorType() {
-    // If the return type of one of the "then" expression or "else" expression is nullable, return "if" expression
-    // type as nullable
+    if (outputType != null) {
+      return outputType;
+    }
+
     MajorType majorType = elseExpression.getMajorType();
     if (majorType.getMinorType() == MinorType.UNION) {
       Set<MinorType> subtypes = Sets.newHashSet();

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/common/src/main/java/org/apache/drill/common/types/Types.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java
index 0c468dc..74b313e 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -35,10 +35,7 @@ public class Types {
   public static final MajorType OPTIONAL_BIT = optional(MinorType.BIT);
 
   public static boolean isUnion(MajorType toType) {
-    if (toType.getMinorType() == MinorType.UNION) {
-      return true;
-    }
-    return false;
+    return toType.getMinorType() == MinorType.UNION;
   }
 
   public static enum Comparability {
@@ -96,7 +93,7 @@ public class Types {
    *   {@code INFORMATION_SCHEMA.COLUMNS.TYPE_NAME} would list)
    */
   public static String getSqlTypeName(final MajorType type) {
-    if (type.getMode() == DataMode.REPEATED) {
+    if (type.getMode() == DataMode.REPEATED || type.getMinorType() == MinorType.LIST) {
       return "ARRAY";
     }
 
@@ -169,7 +166,7 @@ public class Types {
    * Gets JDBC type code for given Drill RPC-/protobuf-level type.
    */
   public static int getJdbcTypeCode(final MajorType type) {
-    if (type.getMode() == DataMode.REPEATED) {
+    if (type.getMode() == DataMode.REPEATED || type.getMinorType() == MinorType.LIST) {
       return java.sql.Types.ARRAY;
     }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
index a4d22cf..2b7b305 100644
--- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
+++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
@@ -33,6 +33,10 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
     super();
   }
 
+  /**
+   * Returns true if the current value of the reader is not null
+   * @return
+   */
   public boolean isSet() {
     return true;
   }
@@ -61,9 +65,7 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
   public void copyAsField(String name, MapWriter writer){
     fail("CopyAsField MapWriter");
   }
-//  public void copyAsValue(ListWriter writer){
-//    fail("CopyAsValueList");
-//  }
+
   public void copyAsField(String name, ListWriter writer){
     fail("CopyAsFieldList");
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
index c3b5cb5..b8b7616 100644
--- a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
+++ b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
@@ -32,19 +32,16 @@ package org.apache.drill.exec.vector.complex.impl;
 @SuppressWarnings("unused")
 public class ComplexCopier {
 
-  FieldReader in;
-  FieldWriter out;
-
-  public ComplexCopier(FieldReader in, FieldWriter out) {
-    this.in = in;
-    this.out = out;
-  }
-
-  public void write() {
-    writeValue(in, out);
+  /**
+   * Do a deep copy of the value in input into output
+   * @param in
+   * @param out
+   */
+  public static void copy(FieldReader input, FieldWriter output) {
+    writeValue(input, output);
   }
 
-  private void writeValue(FieldReader reader, FieldWriter writer) {
+  private static void writeValue(FieldReader reader, FieldWriter writer) {
     final DataMode m = reader.getType().getMode();
     final MinorType mt = reader.getType().getMinorType();
 
@@ -94,7 +91,7 @@ public class ComplexCopier {
     }
  }
 
-  private FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
+  private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
     switch (reader.getType().getMinorType()) {
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />
@@ -113,7 +110,7 @@ public class ComplexCopier {
     }
   }
 
-  private FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
+  private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
     switch (reader.getType().getMinorType()) {
     <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
     <#assign fields = minor.fields!type.fields />

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
index bf447c9..c7676cb 100644
--- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
@@ -119,10 +119,9 @@ public class EventBasedRecordWriter {
   }
 
   public static FieldConverter getConverter(RecordWriter recordWriter, int fieldId, String fieldName, FieldReader reader) {
-    if (reader instanceof UnionReader) {
-      return recordWriter.getNewUnionConverter(fieldId, fieldName, reader);
-    }
     switch (reader.getType().getMinorType()) {
+      case UNION:
+        return recordWriter.getNewUnionConverter(fieldId, fieldName, reader);
       case MAP:
         switch (reader.getType().getMode()) {
           case REQUIRED:
@@ -133,10 +132,7 @@ public class EventBasedRecordWriter {
         }
 
       case LIST:
-        switch (reader.getType().getMode()) {
-          case REPEATED:
-            return recordWriter.getNewRepeatedListConverter(fieldId, fieldName, reader);
-        }
+        return recordWriter.getNewRepeatedListConverter(fieldId, fieldName, reader);
 
         <#list vv.types as type>
         <#list type.minor as minor>

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
index 9bb4f82..d0f5c4a 100644
--- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
+++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
@@ -130,6 +130,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
   <#list fields as field>
     h.${field.name} = holder.${field.name};
   </#list>
+    h.isSet = isSet() ? 1 : 0;
   }
 </#if>
 

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java
index e6becf2..d534571 100644
--- a/exec/java-exec/src/main/codegen/templates/MapWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java
@@ -53,17 +53,15 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
   <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
 
   private final boolean unionEnabled;
-  private final boolean unionInternalMap;
 
-  public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled, boolean unionInternalMap) {
+  public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
     super(parent);
     this.container = container;
     this.unionEnabled = unionEnabled;
-    this.unionInternalMap = unionInternalMap;
   }
 
   public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
-    this(container, parent, false, false);
+    this(container, parent, false);
   }
 
   @Override
@@ -81,11 +79,10 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
       FieldWriter writer = fields.get(name.toLowerCase());
     if(writer == null){
       int vectorCount=container.size();
-      if(!unionEnabled || unionInternalMap){
-        MapVector vector=container.addOrGet(name,MapVector.TYPE,MapVector.class);
-        writer=new SingleMapWriter(vector,this);
-      } else {
         MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
+      if(!unionEnabled){
+        writer = new SingleMapWriter(vector, this);
+      } else {
         writer = new PromotableWriter(vector, container);
       }
       if(vectorCount != container.size()) {
@@ -214,6 +211,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
     FieldWriter writer = fields.get(name.toLowerCase());
     if(writer == null) {
       ValueVector vector;
+      int vectorCount = container.size();
       if (unionEnabled){
         ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
         writer = new PromotableWriter(v, container);
@@ -223,7 +221,9 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
         writer = new ${vectName}WriterImpl(v, this);
         vector = v;
       }
-      vector.allocateNewSafe();
+      if (container.size() > vectorCount) {
+        vector.allocateNewSafe();
+      } 
       writer.setPosition(${index});
       fields.put(name.toLowerCase(), writer);
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullReader.java b/exec/java-exec/src/main/codegen/templates/NullReader.java
index 472dbed..62aa33e 100644
--- a/exec/java-exec/src/main/codegen/templates/NullReader.java
+++ b/exec/java-exec/src/main/codegen/templates/NullReader.java
@@ -60,7 +60,7 @@ public class NullReader extends AbstractBaseReader implements FieldReader{
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   public void read(${name}Holder holder){
-    throw new UnsupportedOperationException("NullReader cannot read into non-nullable holder");
+    throw new UnsupportedOperationException("NullReader cannot write into non-nullable holder");
   }
 
   public void read(Nullable${name}Holder holder){

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/UnionFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java
index 8293d8a..f3ea566 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java
@@ -17,12 +17,12 @@
  */
 
 <@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionFunctions.java" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GUnionFunctions.java" />
 
 
 <#include "/@includes/license.ftl" />
 
-package org.apache.drill.exec.vector.complex.impl;
+package org.apache.drill.exec.expr.fn.impl;
 
 <#include "/@includes/vv_imports.ftl" />
 import org.apache.drill.exec.expr.DrillSimpleFunc;
@@ -40,7 +40,10 @@ import org.apache.drill.exec.record.RecordBatch;
  */
 
 @SuppressWarnings("unused")
-public class UnionFunctions {
+/**
+ * Additional functions can be found in the class UnionFunctions
+ */
+public class GUnionFunctions {
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
@@ -49,7 +52,7 @@ public class UnionFunctions {
   <#if !minor.class?starts_with("Decimal")>
 
   @SuppressWarnings("unused")
-  @FunctionTemplate(name = "is${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  @FunctionTemplate(name = "IS_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
   public static class UnionIs${name} implements DrillSimpleFunc {
 
     @Param UnionHolder in;
@@ -58,12 +61,16 @@ public class UnionFunctions {
     public void setup() {}
 
     public void eval() {
-      out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.${name?upper_case} ? 1 : 0;
+      if (in.isSet == 1) {
+        out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.${name?upper_case} ? 1 : 0;
+      } else {
+        out.value = 0;
+      }
     }
   }
 
   @SuppressWarnings("unused")
-  @FunctionTemplate(name = "as${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  @FunctionTemplate(name = "ASSERT_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
   public static class CastUnion${name} implements DrillSimpleFunc {
 
     @Param UnionHolder in;
@@ -80,21 +87,6 @@ public class UnionFunctions {
     }
   }
 
-  @SuppressWarnings("unused")
-  @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
-  public static class Cast${name}ToUnion implements DrillSimpleFunc {
-
-    @Param Nullable${name}Holder in;
-    @Output UnionHolder out;
-
-    public void setup() {}
-
-    public void eval() {
-      out.reader = new org.apache.drill.exec.vector.complex.impl.Nullable${name}HolderReaderImpl(in);
-      out.isSet = in.isSet;
-    }
-  }
-
   </#if>
 
   </#list></#list>

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
index 35c0bc1..c676769 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import java.lang.UnsupportedOperationException;
+
 <@pp.dropOutputFile />
 <@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionListWriter.java" />
 
@@ -33,9 +35,8 @@ package org.apache.drill.exec.vector.complex.impl;
 @SuppressWarnings("unused")
 public class UnionListWriter extends AbstractFieldWriter {
 
-  ListVector vector;
-//  UnionVector data;
-  UInt4Vector offsets;
+  private ListVector vector;
+  private UInt4Vector offsets;
   private PromotableWriter writer;
   private boolean inMap = false;
   private String mapName;
@@ -44,7 +45,6 @@ public class UnionListWriter extends AbstractFieldWriter {
   public UnionListWriter(ListVector vector) {
     super(null);
     this.vector = vector;
-//    this.data = (UnionVector) vector.getDataVector();
     this.writer = new PromotableWriter(vector.getDataVector(), vector);
     this.offsets = vector.getOffsetVector();
   }
@@ -93,7 +93,11 @@ public class UnionListWriter extends AbstractFieldWriter {
   public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
     assert inMap;
     mapName = name;
-    return this;
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    writer.setPosition(nextOffset);
+    ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
+    return ${uncappedName}Writer;
   }
 
   </#if>
@@ -119,7 +123,6 @@ public class UnionListWriter extends AbstractFieldWriter {
   public ListWriter list(String name) {
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
-//    data.getMutator().setType(nextOffset, MinorType.MAP);
     writer.setPosition(nextOffset);
     ListWriter listWriter = writer.list(name);
     return listWriter;
@@ -146,7 +149,6 @@ public class UnionListWriter extends AbstractFieldWriter {
     assert inMap;
     final int nextOffset = offsets.getAccessor().get(idx() + 1);
     vector.getMutator().setNotNull(idx());
-//    data.getMutator().setType(nextOffset, MinorType.MAP);
     offsets.getMutator().setSafe(idx() + 1, nextOffset);
     writer.setPosition(nextOffset);
   }
@@ -168,20 +170,12 @@ public class UnionListWriter extends AbstractFieldWriter {
 
   @Override
   public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-    if (inMap) {
-      final int nextOffset = offsets.getAccessor().get(idx() + 1);
-      vector.getMutator().setNotNull(idx());
-//      data.getMutator().setType(nextOffset, MinorType.MAP);
-      writer.setPosition(nextOffset);
-      ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(mapName);
-      ${uncappedName}Writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-    } else {
-      final int nextOffset = offsets.getAccessor().get(idx() + 1);
-      vector.getMutator().setNotNull(idx());
-      writer.setPosition(nextOffset);
-      writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-      offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
-    }
+    assert !inMap;
+    final int nextOffset = offsets.getAccessor().get(idx() + 1);
+    vector.getMutator().setNotNull(idx());
+    writer.setPosition(nextOffset);
+    writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
   }
 
   </#if>

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java
index 4943ea0..46a32ee 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionReader.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java
@@ -39,8 +39,16 @@ public class UnionReader extends AbstractFieldReader {
     this.data = data;
   }
 
+  private static MajorType[] TYPES = new MajorType[43];
+
+  static {
+    for (MinorType minorType : MinorType.values()) {
+      TYPES[minorType.getNumber()] = Types.optional(minorType);
+    }
+  }
+
   public MajorType getType() {
-    return Types.required(MinorType.valueOf(data.getTypeValue(idx())));
+    return TYPES[data.getTypeValue(idx())];
   }
 
   public boolean isSet(){
@@ -158,8 +166,7 @@ public class UnionReader extends AbstractFieldReader {
 
   @Override
   public void copyAsValue(ListWriter writer) {
-    ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer);
-    copier.write();
+    ComplexCopier.copy(this, (FieldWriter) writer);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java
index 3e7b3bf..dfcccb2 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionVector.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java
@@ -19,17 +19,18 @@
 import org.apache.drill.common.types.TypeProtos.MinorType;
 
 <@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionVector.java" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/UnionVector.java" />
 
 
 <#include "/@includes/license.ftl" />
 
-package org.apache.drill.exec.vector.complex.impl;
+package org.apache.drill.exec.vector.complex;
 
 <#include "/@includes/vv_imports.ftl" />
 import java.util.Iterator;
 import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
 import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.common.expression.PathSegment;
 
 /*
  * This class is generated using freemarker and the ${.template_name} template.
@@ -37,6 +38,14 @@ import org.apache.drill.exec.util.CallBack;
 @SuppressWarnings("unused")
 
 
+/**
+ * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
+ * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * as well as the addOrGet method.
+ *
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * each time the vector is accessed.
+ */
 public class UnionVector implements ValueVector {
 
   private MaterializedField field;
@@ -46,50 +55,30 @@ public class UnionVector implements ValueVector {
   private int valueCount;
 
   private MapVector internalMap;
-  private SingleMapWriter internalMapWriter;
   private UInt1Vector typeVector;
 
   private MapVector mapVector;
   private ListVector listVector;
-  private NullableBigIntVector bigInt;
-  private NullableVarCharVector varChar;
 
   private FieldReader reader;
   private NullableBitVector bit;
 
-  private State state = State.INIT;
   private int singleType = 0;
   private ValueVector singleVector;
   private MajorType majorType;
 
   private final CallBack callBack;
 
-  private enum State {
-    INIT, SINGLE, MULTI
-  }
-
   public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
     this.field = field.clone();
     this.allocator = allocator;
-    internalMap = new MapVector("internal", allocator, callBack);
-    internalMapWriter = new SingleMapWriter(internalMap, null, true, true);
+    this.internalMap = new MapVector("internal", allocator, callBack);
     this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class);
     this.field.addChild(internalMap.getField().clone());
     this.majorType = field.getType();
     this.callBack = callBack;
   }
 
-  private void updateState(ValueVector v) {
-    if (state == State.INIT) {
-      state = State.SINGLE;
-      singleVector = v;
-      singleType = v.getField().getType().getMinorType().getNumber();
-    } else {
-      state = State.MULTI;
-      singleVector = null;
-    }
-  }
-
   public List<MinorType> getSubTypes() {
     return majorType.getSubTypeList();
   }
@@ -101,21 +90,12 @@ public class UnionVector implements ValueVector {
     }
   }
 
-  public boolean isSingleType() {
-    return state == State.SINGLE && singleType != MinorType.LIST_VALUE;
-  }
-
-  public ValueVector getSingleVector() {
-    assert state != State.MULTI : "Cannot get single vector when there are multiple types";
-    assert state != State.INIT : "Cannot get single vector when there are no types";
-    return singleVector;
-  }
+  private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP);
 
   public MapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
-      mapVector = internalMap.addOrGet("map", Types.optional(MinorType.MAP), MapVector.class);
-      updateState(mapVector);
+      mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
       addSubType(MinorType.MAP);
       if (internalMap.size() > vectorCount) {
         mapVector.allocateNew();
@@ -130,12 +110,12 @@ public class UnionVector implements ValueVector {
   <#if !minor.class?starts_with("Decimal")>
 
   private Nullable${name}Vector ${uncappedName}Vector;
+  private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case});
 
   public Nullable${name}Vector get${name}Vector() {
     if (${uncappedName}Vector == null) {
       int vectorCount = internalMap.size();
-      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", Types.optional(MinorType.${name?upper_case}), Nullable${name}Vector.class);
-      updateState(${uncappedName}Vector);
+      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
       addSubType(MinorType.${name?upper_case});
       if (internalMap.size() > vectorCount) {
         ${uncappedName}Vector.allocateNew();
@@ -148,11 +128,12 @@ public class UnionVector implements ValueVector {
 
   </#list></#list>
 
+  private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST);
+
   public ListVector getList() {
     if (listVector == null) {
       int vectorCount = internalMap.size();
-      listVector = internalMap.addOrGet("list", Types.optional(MinorType.LIST), ListVector.class);
-      updateState(listVector);
+      listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
       addSubType(MinorType.LIST);
       if (internalMap.size() > vectorCount) {
         listVector.allocateNew();
@@ -235,8 +216,7 @@ public class UnionVector implements ValueVector {
   public void copyFrom(int inIndex, int outIndex, UnionVector from) {
     from.getReader().setPosition(inIndex);
     getWriter().setPosition(outIndex);
-    ComplexCopier copier = new ComplexCopier(from.reader, mutator.writer);
-    copier.write();
+    ComplexCopier.copy(from.reader, mutator.writer);
   }
 
   public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
@@ -244,7 +224,9 @@ public class UnionVector implements ValueVector {
   }
 
   public void addVector(ValueVector v) {
-    internalMap.putChild(v.getField().getType().getMinorType().name().toLowerCase(), v);
+    String name = v.getField().getType().getMinorType().name().toLowerCase();
+    Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
+    internalMap.putChild(name, v);
     addSubType(v.getField().getType().getMinorType());
   }
 
@@ -355,6 +337,24 @@ public class UnionVector implements ValueVector {
     return vectors.iterator();
   }
 
+  public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+    if (seg.isNamed()) {
+      ValueVector v = getMap();
+      if (v != null) {
+        return ((AbstractContainerVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+      } else {
+        return null;
+      }
+    } else if (seg.isArray()) {
+      ValueVector v = getList();
+      if (v != null) {
+        return ((ListVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+      }
+      else return null;
+    }
+    return null;
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
 
@@ -422,9 +422,6 @@ public class UnionVector implements ValueVector {
       internalMap.getMutator().setValueCount(valueCount);
     }
 
-    public void set(int index, byte[] bytes) {
-    }
-
     public void setSafe(int index, UnionHolder holder) {
       FieldReader reader = holder.reader;
       if (writer == null) {
@@ -440,20 +437,16 @@ public class UnionVector implements ValueVector {
       case ${name?upper_case}:
         Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
         reader.read(${uncappedName}Holder);
-        if (holder.isSet == 1) {
-          writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list>);
-        }
+        setSafe(index, ${uncappedName}Holder);
         break;
       </#if>
       </#list></#list>
       case MAP: {
-        ComplexCopier copier = new ComplexCopier(reader, writer);
-        copier.write();
+        ComplexCopier.copy(reader, writer);
         break;
       }
       case LIST: {
-        ComplexCopier copier = new ComplexCopier(reader, writer);
-        copier.write();
+        ComplexCopier.copy(reader, writer);
         break;
       }
       default:
@@ -461,6 +454,18 @@ public class UnionVector implements ValueVector {
       }
     }
 
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign fields = minor.fields!type.fields />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    public void setSafe(int index, Nullable${name}Holder holder) {
+      setType(index, MinorType.${name?upper_case});
+      get${name}Vector().getMutator().setSafe(index, holder);
+    }
+
+    </#if>
+    </#list></#list>
+
     public void setType(int index, MinorType type) {
       typeVector.getMutator().setSafe(index, type.getNumber());
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionWriter.java b/exec/java-exec/src/main/codegen/templates/UnionWriter.java
index 9d77999..7a123b4 100644
--- a/exec/java-exec/src/main/codegen/templates/UnionWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/UnionWriter.java
@@ -63,28 +63,28 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
   @Override
   public void start() {
     data.getMutator().setType(idx(), MinorType.MAP);
-    getMapWriter(true).start();
+    getMapWriter().start();
   }
 
   @Override
   public void end() {
-    getMapWriter(false).end();
+    getMapWriter().end();
   }
 
   @Override
   public void startList() {
-    getListWriter(true).startList();
+    getListWriter().startList();
     data.getMutator().setType(idx(), MinorType.LIST);
   }
 
   @Override
   public void endList() {
-    getListWriter(true).endList();
+    getListWriter().endList();
   }
 
-  private MapWriter getMapWriter(boolean create) {
-    if (create && mapWriter == null) {
-      mapWriter = new SingleMapWriter(data.getMap(), null, true, false);
+  private MapWriter getMapWriter() {
+    if (mapWriter == null) {
+      mapWriter = new SingleMapWriter(data.getMap(), null, true);
       mapWriter.setPosition(idx());
       writers.add(mapWriter);
     }
@@ -93,11 +93,11 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
 
   public MapWriter asMap() {
     data.getMutator().setType(idx(), MinorType.MAP);
-    return getMapWriter(true);
+    return getMapWriter();
   }
 
-  private ListWriter getListWriter(boolean create) {
-    if (create && listWriter == null) {
+  private ListWriter getListWriter() {
+    if (listWriter == null) {
       listWriter = new UnionListWriter(data.getList());
       listWriter.setPosition(idx());
       writers.add(listWriter);
@@ -107,7 +107,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
 
   public ListWriter asList() {
     data.getMutator().setType(idx(), MinorType.LIST);
-    return getListWriter(true);
+    return getListWriter();
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
@@ -118,8 +118,8 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
 
   private ${name}Writer ${name?uncap_first}Writer;
 
-  private ${name}Writer get${name}Writer(boolean create) {
-    if (create && ${uncappedName}Writer == null) {
+  private ${name}Writer get${name}Writer() {
+    if (${uncappedName}Writer == null) {
       ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
       ${uncappedName}Writer.setPosition(idx());
       writers.add(${uncappedName}Writer);
@@ -129,20 +129,20 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
 
   public ${name}Writer as${name}() {
     data.getMutator().setType(idx(), MinorType.${name?upper_case});
-    return get${name}Writer(true);
+    return get${name}Writer();
   }
 
   @Override
   public void write(${name}Holder holder) {
     data.getMutator().setType(idx(), MinorType.${name?upper_case});
-    get${name}Writer(true).setPosition(idx());
-    get${name}Writer(true).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
   }
 
   public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
     data.getMutator().setType(idx(), MinorType.${name?upper_case});
-    get${name}Writer(true).setPosition(idx());
-    get${name}Writer(true).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
   }
   </#if>
 
@@ -154,29 +154,29 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
   @Override
   public MapWriter map() {
     data.getMutator().setType(idx(), MinorType.LIST);
-    getListWriter(true).setPosition(idx());
-    return getListWriter(true).map();
+    getListWriter().setPosition(idx());
+    return getListWriter().map();
   }
 
   @Override
   public ListWriter list() {
     data.getMutator().setType(idx(), MinorType.LIST);
-    getListWriter(true).setPosition(idx());
-    return getListWriter(true).list();
+    getListWriter().setPosition(idx());
+    return getListWriter().list();
   }
 
   @Override
   public ListWriter list(String name) {
     data.getMutator().setType(idx(), MinorType.MAP);
-    getMapWriter(true).setPosition(idx());
-    return getMapWriter(true).list(name);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().list(name);
   }
 
   @Override
   public MapWriter map(String name) {
     data.getMutator().setType(idx(), MinorType.MAP);
-    getMapWriter(true).setPosition(idx());
-    return getMapWriter(true).map(name);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().map(name);
   }
 
   <#list vv.types as type><#list type.minor as minor>
@@ -188,15 +188,15 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
   @Override
   public ${capName}Writer ${lowerName}(String name) {
     data.getMutator().setType(idx(), MinorType.MAP);
-    getMapWriter(true).setPosition(idx());
-    return getMapWriter(true).${lowerName}(name);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().${lowerName}(name);
   }
 
   @Override
   public ${capName}Writer ${lowerName}() {
     data.getMutator().setType(idx(), MinorType.LIST);
-    getListWriter(true).setPosition(idx());
-    return getListWriter(true).${lowerName}();
+    getListWriter().setPosition(idx());
+    return getListWriter().${lowerName}();
   }
   </#if>
   </#list></#list>

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
index e4fa260..b1fb5e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
@@ -49,6 +49,9 @@ import org.apache.drill.exec.expr.fn.DrillFuncHolder;
 
 import java.util.List;
 
+/**
+ * Creates a deep copy of a LogicalExpression. Specifically, it creates new instances of the literal expressions
+ */
 public class CloneVisitor extends AbstractExprVisitor<LogicalExpression,Void,RuntimeException> {
   @Override
   public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 424174e..0c65835 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -351,10 +351,10 @@ public class EvaluationVisitor {
       // evaluation work.
       HoldingContainer out = generator.declare(e.getMajorType());
 
-      final boolean primitive = !Types.usesHolderForGet(e.getMajorType());
       final boolean hasReadPath = e.hasReadPath();
       final boolean complex = Types.isComplex(e.getMajorType());
       final boolean repeated = Types.isRepeated(e.getMajorType());
+      final boolean listVector = e.getTypedFieldId().isListVector();
 
       int[] fieldIds = e.getFieldId().getFieldIds();
       for (int i = 1; i < fieldIds.length; i++) {
@@ -372,7 +372,7 @@ public class EvaluationVisitor {
         PathSegment seg = e.getReadPath();
 
         JVar isNull = null;
-        boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated);
+        boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated || listVector);
         if (isNullReaderLikely) {
           isNull = generator.getEvalBlock().decl(generator.getModel().INT, generator.getNextVar("isNull"), JExpr.lit(0));
         }
@@ -389,7 +389,7 @@ public class EvaluationVisitor {
           if (seg.isArray()) {
             // stop once we get to the last segment and the final type is neither complex nor repeated (map, list, repeated list).
             // In case of non-complex and non-repeated type, we return Holder, in stead of FieldReader.
-            if (seg.isLastPath() && !complex && !repeated && !e.getTypedFieldId().isListVector()) {
+            if (seg.isLastPath() && !complex && !repeated && !listVector) {
               break;
             }
 
@@ -413,7 +413,7 @@ public class EvaluationVisitor {
             if (out.isOptional()) {
               ifNoVal.assign(out.getIsSet(), JExpr.lit(0));
             }
-            ifNoVal.assign(isNull,  JExpr.lit(1));
+            ifNoVal.assign(isNull, JExpr.lit(1));
             ifNoVal._break(label);
 
             expr = list.invoke("reader");

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 827623b..daac31d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -17,16 +17,18 @@
  */
 package org.apache.drill.exec.expr;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
+import java.util.ArrayDeque;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Queue;
+import java.util.Set;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.BooleanOperator;
 import org.apache.drill.common.expression.CastExpression;
@@ -39,7 +41,6 @@ import org.apache.drill.common.expression.FunctionHolderExpression;
 import org.apache.drill.common.expression.IfExpression;
 import org.apache.drill.common.expression.IfExpression.IfCondition;
 import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.LogicalExpressionBase;
 import org.apache.drill.common.expression.NullExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.TypedNullConstant;
@@ -62,6 +63,7 @@ import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
 import org.apache.drill.common.expression.fn.CastFunctions;
 import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
 import org.apache.drill.common.expression.visitors.ConditionalExprOptimizer;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
 import org.apache.drill.common.expression.visitors.ExpressionValidator;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.DataMode;
@@ -74,7 +76,7 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.fn.AbstractFuncHolder;
 import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
 import org.apache.drill.exec.expr.fn.DrillFuncHolder;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.ExceptionFunction;
 import org.apache.drill.exec.expr.fn.FunctionLookupContext;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorAccessible;
@@ -109,6 +111,11 @@ public class ExpressionTreeMaterializer {
   }
 
   public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext,
+                                              boolean allowComplexWriterExpr) {
+    return materialize(expr, batch, errorCollector, functionLookupContext, allowComplexWriterExpr, false);
+  }
+
+  public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext,
       boolean allowComplexWriterExpr, boolean unionTypeEnabled) {
     LogicalExpression out =  expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr, unionTypeEnabled), functionLookupContext);
 
@@ -139,12 +146,19 @@ public class ExpressionTreeMaterializer {
     return matchedConvertToNullableFuncHolder.getExpr(funcName, args, ExpressionPosition.UNKNOWN);
   }
 
-
   public static LogicalExpression addCastExpression(LogicalExpression fromExpr, MajorType toType, FunctionLookupContext functionLookupContext, ErrorCollector errorCollector) {
+    return addCastExpression(fromExpr, toType, functionLookupContext, errorCollector, true);
+  }
+
+  public static LogicalExpression addCastExpression(LogicalExpression fromExpr, MajorType toType, FunctionLookupContext functionLookupContext, ErrorCollector errorCollector, boolean exactResolver) {
     String castFuncName = CastFunctions.getCastFunc(toType.getMinorType());
     List<LogicalExpression> castArgs = Lists.newArrayList();
     castArgs.add(fromExpr);  //input_expr
 
+    if (fromExpr.getMajorType().getMinorType() == MinorType.UNION && toType.getMinorType() == MinorType.UNION) {
+      return fromExpr;
+    }
+
     if (!Types.isFixedWidthType(toType) && !Types.isUnion(toType)) {
 
       /* We are implicitly casting to VARCHAR so we don't have a max length,
@@ -159,7 +173,12 @@ public class ExpressionTreeMaterializer {
       castArgs.add(new ValueExpressions.LongExpression(toType.getScale(), null));
     }
     FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
-    FunctionResolver resolver = FunctionResolverFactory.getExactResolver(castCall);
+    FunctionResolver resolver;
+    if (exactResolver) {
+      resolver = FunctionResolverFactory.getExactResolver(castCall);
+    } else {
+      resolver = FunctionResolverFactory.getResolver(castCall);
+    }
     DrillFuncHolder matchedCastFuncHolder = functionLookupContext.findDrillFunction(resolver, castCall);
 
     if (matchedCastFuncHolder == null) {
@@ -194,12 +213,22 @@ public class ExpressionTreeMaterializer {
     errorCollector.addGeneralError(call.getPosition(), sb.toString());
   }
 
+
   private static class MaterializeVisitor extends AbstractExprVisitor<LogicalExpression, FunctionLookupContext, RuntimeException> {
     private ExpressionValidator validator = new ExpressionValidator();
-    private final ErrorCollector errorCollector;
+    private ErrorCollector errorCollector;
+    private Deque<ErrorCollector> errorCollectors = new ArrayDeque<>();
     private final VectorAccessible batch;
     private final boolean allowComplexWriter;
+    /**
+     * If this is false, the materializer will not handle or create UnionTypes
+     * Once this code is more well tested, we will probably remove this flag
+     */
     private final boolean unionTypeEnabled;
+    /**
+     * Avoid revisiting portions of the tree that have already been materialized
+     */
+    private Set<LogicalExpression> materializedExpressions = Sets.newIdentityHashSet();
 
     public MaterializeVisitor(VectorAccessible batch, ErrorCollector errorCollector, boolean allowComplexWriter, boolean unionTypeEnabled) {
       this.batch = batch;
@@ -250,10 +279,6 @@ public class ExpressionTreeMaterializer {
       //replace with a new function call, since its argument could be changed.
       call = new FunctionCall(call.getName(), args, call.getPosition());
 
-      if (hasUnionInput(call)) {
-        return rewriteUnionFunction(call, functionLookupContext);
-      }
-
       FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
       DrillFuncHolder matchedFuncHolder = functionLookupContext.findDrillFunction(resolver, call);
 
@@ -323,21 +348,42 @@ public class ExpressionTreeMaterializer {
         return matchedNonDrillFuncHolder.getExpr(call.getName(), extArgsWithCast, call.getPosition());
       }
 
+      if (hasUnionInput(call)) {
+        return rewriteUnionFunction(call, functionLookupContext);
+      }
+
       logFunctionResolutionError(errorCollector, call);
       return NullExpression.INSTANCE;
     }
 
+    private static final Set<String> UNION_FUNCTIONS;
+
+    static {
+      UNION_FUNCTIONS = new HashSet<>();
+      for (MinorType t : MinorType.values()) {
+        UNION_FUNCTIONS.add("assert_" + t.name().toLowerCase());
+        UNION_FUNCTIONS.add("is_" + t.name().toLowerCase());
+      }
+      UNION_FUNCTIONS.add("typeof");
+    }
+
     private boolean hasUnionInput(FunctionCall call) {
       for (LogicalExpression arg : call.args) {
         if (arg.getMajorType().getMinorType() == MinorType.UNION) {
-          if (!call.getName().toLowerCase().startsWith("as") && !call.getName().toLowerCase().startsWith("is") && !call.getName().toLowerCase().startsWith("typeof")) {
-            return true;
-          }
+          return true;
         }
       }
       return false;
     }
 
+    /**
+     * Converts a function call with a Union type input into a case statement, where each branch of the case corresponds to
+     * one of the subtypes of the Union type. The function call is materialized in each of the branches, with the union input cast
+     * to the specific type corresponding to the branch of the case statement
+     * @param call
+     * @param functionLookupContext
+     * @return
+     */
     private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookupContext functionLookupContext) {
       LogicalExpression[] args = new LogicalExpression[call.args.size()];
       call.args.toArray(args);
@@ -357,14 +403,29 @@ public class ExpressionTreeMaterializer {
 
         for (MinorType minorType : subTypes) {
           LogicalExpression ifCondition = getIsTypeExpressionForType(minorType, arg.accept(new CloneVisitor(), null));
-          args[i] = getUnionCastExpressionForType(minorType, arg.accept(new CloneVisitor(), null));
+          args[i] = getUnionAssertFunctionForType(minorType, arg.accept(new CloneVisitor(), null));
 
           List<LogicalExpression> newArgs = Lists.newArrayList();
           for (LogicalExpression e : args) {
             newArgs.add(e.accept(new CloneVisitor(), null));
           }
 
-          LogicalExpression thenExpression = new FunctionCall(call.getName(), newArgs, call.getPosition());
+          // When expanding the expression tree to handle the different subtypes, we will not throw an exception if one
+          // of the branches fails to find a function match, since it is possible that code path will never occur in execution
+          // So instead of failing to materialize, we generate code to throw the exception during execution if that code
+          // path is hit.
+
+          errorCollectors.push(errorCollector);
+          errorCollector = new ErrorCollectorImpl();
+
+          LogicalExpression thenExpression = new FunctionCall(call.getName(), newArgs, call.getPosition()).accept(this, functionLookupContext);
+
+          if (errorCollector.hasErrors()) {
+            thenExpression = getExceptionFunction(errorCollector.toErrorString());
+          }
+
+          errorCollector = errorCollectors.pop();
+
           IfExpression.IfCondition condition = new IfCondition(ifCondition, thenExpression);
           ifConditions.add(condition);
         }
@@ -381,24 +442,51 @@ public class ExpressionTreeMaterializer {
       throw new UnsupportedOperationException("Did not find any Union input types");
     }
 
-    private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) {
+    /**
+     * Returns the function call whose purpose is to throw an Exception if that code is hit during execution
+     * @param message the exception message
+     * @return
+     */
+    private LogicalExpression getExceptionFunction(String message) {
+      QuotedString msg = new QuotedString(message, ExpressionPosition.UNKNOWN);
+      List<LogicalExpression> args = Lists.newArrayList();
+      args.add(msg);
+      FunctionCall call = new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN);
+      return call;
+    }
+
+    /**
+     * Returns the function which asserts that the current subtype of a union type is a specific type, and allows the materializer
+     * to bind to that specific type when doing function resolution
+     * @param type
+     * @param arg
+     * @return
+     */
+    private LogicalExpression getUnionAssertFunctionForType(MinorType type, LogicalExpression arg) {
       if (type == MinorType.UNION) {
-        return null;
+        return arg;
       }
-      String castFuncName = String.format("as%s", type.toString());
-      List<LogicalExpression> args = Lists.newArrayList();
-      args.add(arg);
-      return new FunctionCall(castFuncName, args, ExpressionPosition.UNKNOWN);
+      if (type == MinorType.LIST || type == MinorType.MAP) {
+        return getExceptionFunction("Unable to cast union to " + type);
+      }
+      String castFuncName = String.format("assert_%s", type.toString());
+      Collections.singletonList(arg);
+      return new FunctionCall(castFuncName, Collections.singletonList(arg), ExpressionPosition.UNKNOWN);
     }
 
+    /**
+     * Get the function that tests whether a union type is a specific type
+     * @param type
+     * @param arg
+     * @return
+     */
     private LogicalExpression getIsTypeExpressionForType(MinorType type, LogicalExpression arg) {
-      String isFuncName = String.format("is%s", type.toString());
+      String isFuncName = String.format("is_%s", type.toString());
       List<LogicalExpression> args = Lists.newArrayList();
       args.add(arg);
       return new FunctionCall(isFuncName, args, ExpressionPosition.UNKNOWN);
     }
 
-    @Override
     public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupContext functionLookupContext) {
       IfExpression.IfCondition conditions = ifExpr.ifCondition;
       LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, functionLookupContext);
@@ -410,6 +498,7 @@ public class ExpressionTreeMaterializer {
       MinorType thenType = conditions.expression.getMajorType().getMinorType();
       MinorType elseType = newElseExpr.getMajorType().getMinorType();
       boolean hasUnion = thenType == MinorType.UNION || elseType == MinorType.UNION;
+      MajorType outputType = ifExpr.outputType;
       if (unionTypeEnabled) {
         if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) {
 
@@ -429,10 +518,10 @@ public class ExpressionTreeMaterializer {
           } else {
             builder.addSubType(elseType);
           }
-          MajorType unionType = builder.build();
+          outputType = builder.build();
           conditions = new IfExpression.IfCondition(newCondition,
-                  addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector));
-          newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector);
+                  addCastExpression(conditions.expression, outputType, functionLookupContext, errorCollector, false));
+          newElseExpr = addCastExpression(newElseExpr, outputType, functionLookupContext, errorCollector, false);
         }
 
       } else {
@@ -505,7 +594,8 @@ public class ExpressionTreeMaterializer {
         }
       }
 
-      return validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).build());
+      LogicalExpression expr = validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).setOutputType(outputType).build());
+      return expr;
     }
 
     private LogicalExpression getConvertToNullableExpr(List<LogicalExpression> args, MinorType minorType,
@@ -663,7 +753,7 @@ public class ExpressionTreeMaterializer {
         String castFuncWithType = CastFunctions.getCastFunc(type.getMinorType());
 
         List<LogicalExpression> newArgs = Lists.newArrayList();
-        newArgs.add(e.getInput());  //input_expr
+        newArgs.add(input);  //input_expr
 
         //VarLen type
         if (!Types.isFixedWidthType(type)) {

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java
new file mode 100644
index 0000000..9e5fced
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
+public class ExceptionFunction {
+
+  public static final String EXCEPTION_FUNCTION_NAME = "__throwException";
+
+  @FunctionTemplate(name = EXCEPTION_FUNCTION_NAME, isRandom = true,
+          scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+  public static class ThrowException implements DrillSimpleFunc {
+
+    @Param VarCharHolder message;
+    @Output BigIntHolder out;
+
+    public void setup() {
+    }
+
+    public void eval() {
+      String msg = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(message.start, message.end, message.buffer);
+      org.apache.drill.exec.expr.fn.ExceptionFunction.throwException(msg);
+    }
+  }
+
+  public static void throwException(String message) {
+    throw new org.apache.drill.common.exceptions.DrillRuntimeException(message);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
index 34b14fe..26cc43b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java
@@ -18,12 +18,16 @@
 package org.apache.drill.exec.expr.fn.impl;
 
 import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
 import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTinyIntHolder;
 import org.apache.drill.exec.expr.holders.NullableUInt1Holder;
 import org.apache.drill.exec.expr.holders.UnionHolder;
 import org.apache.drill.exec.expr.holders.IntHolder;
@@ -33,92 +37,99 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
 
 import javax.inject.Inject;
 
+/**
+ * The class contains additional functions for union types in addition to those in GUnionFunctions
+ */
 public class UnionFunctions {
 
-  @FunctionTemplate(names = {"typeString"},
+  @FunctionTemplate(names = {"typeOf"},
           scope = FunctionTemplate.FunctionScope.SIMPLE,
-          nulls = NullHandling.NULL_IF_NULL)
-  public static class FromType implements DrillSimpleFunc {
+          nulls = NullHandling.INTERNAL)
+  public static class GetType implements DrillSimpleFunc {
 
     @Param
-    IntHolder in;
+    FieldReader input;
     @Output
     VarCharHolder out;
     @Inject
-    DrillBuf buffer;
+    DrillBuf buf;
 
     public void setup() {}
 
     public void eval() {
 
-      VarCharHolder h = org.apache.drill.exec.vector.ValueHolderHelper.getVarCharHolder(buffer, org.apache.drill.common.types.MinorType.valueOf(in.value).toString());
-      out.buffer = h.buffer;
-      out.start = h.start;
-      out.end = h.end;
+      byte[] type;
+      if (input.isSet()) {
+         type = input.getType().getMinorType().name().getBytes();
+      } else {
+        type = org.apache.drill.common.types.TypeProtos.MinorType.NULL.name().getBytes();
+      }
+      buf = buf.reallocIfNeeded(type.length);
+      buf.setBytes(0, type);
+      out.buffer = buf;
+      out.start = 0;
+      out.end = type.length;
     }
   }
 
-  @FunctionTemplate(names = {"type"},
-          scope = FunctionTemplate.FunctionScope.SIMPLE,
-          nulls = NullHandling.NULL_IF_NULL)
-  public static class ToType implements DrillSimpleFunc {
+  @SuppressWarnings("unused")
+  @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
+  public static class CastUnionToUnion implements DrillSimpleFunc{
 
-    @Param
-    VarCharHolder input;
+    @Param FieldReader in;
     @Output
-    IntHolder out;
+    UnionHolder out;
 
     public void setup() {}
 
     public void eval() {
-
-      out.value = input.getType().getMinorType().getNumber();
-      byte[] b = new byte[input.end - input.start];
-      input.buffer.getBytes(input.start, b, 0, b.length);
-      String type = new String(b);
-      out.value = org.apache.drill.common.types.MinorType.valueOf(type.toUpperCase()).getNumber();
+      out.reader = in;
+      out.isSet = in.isSet() ? 1 : 0;
     }
   }
 
-  @FunctionTemplate(names = {"typeOf"},
-          scope = FunctionTemplate.FunctionScope.SIMPLE,
-          nulls = NullHandling.INTERNAL)
-  public static class GetType implements DrillSimpleFunc {
+  @SuppressWarnings("unused")
+  @FunctionTemplate(name = "ASSERT_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  public static class CastUnionList implements DrillSimpleFunc {
 
-    @Param
-    FieldReader input;
-    @Output
-    NullableIntHolder out;
+    @Param UnionHolder in;
+    @Output UnionHolder out;
 
     public void setup() {}
 
     public void eval() {
-
-      out.value = input.isSet() ? input.getType().getMinorType().getNumber() : 0;
-      out.isSet = 1;
-
+      if (in.isSet == 1) {
+        if (in.reader.getType().getMinorType() != org.apache.drill.common.types.TypeProtos.MinorType.LIST) {
+          throw new UnsupportedOperationException("The input is not a LIST type");
+        }
+        out.reader = in.reader;
+      } else {
+        out.isSet = 0;
+      }
     }
   }
 
   @SuppressWarnings("unused")
-  @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
-  public static class CastUnionToUnion implements DrillSimpleFunc{
+  @FunctionTemplate(name = "IS_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  public static class UnionIsList implements DrillSimpleFunc {
 
-    @Param FieldReader in;
-    @Output
-    UnionHolder out;
+    @Param UnionHolder in;
+    @Output BitHolder out;
 
     public void setup() {}
 
     public void eval() {
-      out.reader = in;
-      out.isSet = in.isSet() ? 1 : 0;
+      if (in.isSet == 1) {
+        out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.LIST ? 1 : 0;
+      } else {
+        out.value = 0;
+      }
     }
   }
 
   @SuppressWarnings("unused")
-  @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
-  public static class CastUnionList implements DrillSimpleFunc {
+  @FunctionTemplate(name = "ASSERT_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  public static class CastUnionMap implements DrillSimpleFunc {
 
     @Param UnionHolder in;
     @Output UnionHolder out;
@@ -127,10 +138,58 @@ public class UnionFunctions {
 
     public void eval() {
       if (in.isSet == 1) {
+        if (in.reader.getType().getMinorType() != org.apache.drill.common.types.TypeProtos.MinorType.MAP) {
+          throw new UnsupportedOperationException("The input is not a MAP type");
+        }
         out.reader = in.reader;
       } else {
         out.isSet = 0;
       }
     }
   }
+
+  @SuppressWarnings("unused")
+  @FunctionTemplate(name = "IS_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+  public static class UnionIsMap implements DrillSimpleFunc {
+
+    @Param UnionHolder in;
+    @Output BitHolder out;
+
+    public void setup() {}
+
+    public void eval() {
+      if (in.isSet == 1) {
+        out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.MAP ? 1 : 0;
+      } else {
+        out.value = 0;
+      }
+    }
+  }
+
+  @FunctionTemplate(names = {"isnotnull", "is not null"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+  public static class IsNotNull implements DrillSimpleFunc {
+
+    @Param UnionHolder input;
+    @Output BitHolder out;
+
+    public void setup() { }
+
+    public void eval() {
+      out.value = input.isSet == 1 ? 1 : 0;
+    }
+  }
+
+  @FunctionTemplate(names = {"isnull", "is null"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+  public static class IsNull implements DrillSimpleFunc {
+
+    @Param UnionHolder input;
+    @Output BitHolder out;
+
+    public void setup() { }
+
+    public void eval() {
+      out.value = input.isSet == 1 ? 0 : 1;
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
index 98b2876..e7ee84d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
@@ -90,6 +90,7 @@ public class ImplCreator {
 
       return rootExec;
     } catch(Exception e) {
+      e.printStackTrace();
       context.fail(e);
       for(final CloseableRecordBatch crb : creator.getOperators()) {
         AutoCloseables.close(crb, logger);

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index e205246..8a7d705 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -213,7 +213,7 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
           ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
 
       if (expr instanceof IfExpression) {
-        throw new SchemaChangeException("Union type not supported in aggregate functions");
+        throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
       }
 
       if (collector.hasErrors()) {

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index 9f92d22..89122c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -282,7 +282,7 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
       final NamedExpression ne = popConfig.getExprs()[i];
       final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
       if (expr instanceof IfExpression) {
-        throw new SchemaChangeException("Union type not supported in aggregate functions");
+        throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
       }
       if (expr == null) {
         continue;

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
index fd16bc8..0a96285 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
@@ -22,6 +22,7 @@ import java.util.List;
 
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionStringBuilder;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.exception.ClassTransformationException;

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 9f64626..3a8b735 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -288,7 +288,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
     final IntOpenHashSet transferFieldIds = new IntOpenHashSet();
 
     final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
-    final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false);
+    final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
     final TransferPair tp = getFlattenFieldTransferPair(flattenExpr.getRef());
 
     if (tp != null) {
@@ -315,7 +315,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
         }
       }
 
-      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false);
+      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
       final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType());
       if (collector.hasErrors()) {
         throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
index 1687c3d..ed7da9b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
@@ -134,7 +134,6 @@ public class IteratorValidatorBatchIterator implements CloseableRecordBatch {
         throw new IllegalStateException (String.format("Incoming batch of %s has size %d, which is beyond the limit of %d",  incoming.getClass().getName(), incoming.getRecordCount(), MAX_BATCH_SIZE));
       }
 
-//      BatchPrinter.printBatch(incoming);
       if (VALIDATE_VECTORS) {
         VectorValidator.validate(incoming);
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
index 2a8e91f..85ec466 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
@@ -31,7 +31,7 @@ import org.apache.drill.exec.vector.complex.AbstractContainerVector;
 import org.apache.drill.exec.vector.complex.AbstractMapVector;
 import org.apache.drill.exec.vector.complex.ListVector;
 import org.apache.drill.exec.vector.complex.MapVector;
-import org.apache.drill.exec.vector.complex.impl.UnionVector;
+import org.apache.drill.exec.vector.complex.UnionVector;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -123,9 +123,13 @@ public class SimpleVectorWrapper<T extends ValueVector> implements VectorWrapper
         majorTypeBuilder.addSubType(type);
       }
       MajorType majorType = majorTypeBuilder.build();
-      builder.finalType(majorType);
       builder.intermediateType(majorType);
-      return builder.build();
+      if (seg.isLastPath()) {
+        builder.finalType(majorType);
+        return builder.build();
+      } else {
+        return ((UnionVector) v).getFieldIdIfMatches(builder, false, seg.getChild());
+      }
     } else if (v instanceof ListVector) {
       ListVector list = (ListVector) v;
       TypedFieldId.Builder builder = TypedFieldId.newBuilder();

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java
index ecf2596..8091c56 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.vector.accessor;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.impl.UnionVector;
+import org.apache.drill.exec.vector.complex.UnionVector;
 import org.apache.drill.exec.vector.complex.impl.UnionWriter;
 import org.apache.drill.exec.vector.complex.reader.FieldReader;
 

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 17b42a1..5f6d0d2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -111,84 +111,10 @@ public abstract class AbstractContainerVector implements ValueVector {
   }
 
   public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
-    if (seg == null) {
-      if (addToBreadCrumb) {
-        builder.intermediateType(this.getField().getType());
-      }
-      return builder.finalType(this.getField().getType()).build();
-    }
-
-    if (seg.isArray()) {
-      if (seg.isLastPath()) {
-        builder //
-          .withIndex() //
-          .finalType(getLastPathType());
-
-        // remainder starts with the 1st array segment in SchemaPath.
-        // only set remainder when it's the only array segment.
-        if (addToBreadCrumb) {
-          addToBreadCrumb = false;
-          builder.remainder(seg);
-        }
-        return builder.build();
-      } else {
-        if (addToBreadCrumb) {
-          addToBreadCrumb = false;
-          builder.remainder(seg);
-        }
-      }
-    } else {
-      // name segment.
-    }
-
-    VectorWithOrdinal vord = getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath());
-    if (vord == null) {
-      return null;
-    }
-
-    ValueVector v = vord.vector;
-    if (addToBreadCrumb) {
-      builder.intermediateType(v.getField().getType());
-      builder.addId(vord.ordinal);
-    }
-
-    if (v instanceof AbstractContainerVector) {
-      // we're looking for a multi path.
-      AbstractContainerVector c = (AbstractContainerVector) v;
-      return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
-    } else if(v instanceof ListVector) {
-      ListVector list = (ListVector) v;
-      return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
-    } else {
-      if (seg.isNamed()) {
-        if(addToBreadCrumb) {
-          builder.intermediateType(v.getField().getType());
-        }
-        builder.finalType(v.getField().getType());
-      } else {
-        builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
-      }
-
-      if (seg.isLastPath()) {
-        return builder.build();
-      } else {
-        PathSegment child = seg.getChild();
-        if (child.isLastPath() && child.isArray()) {
-          if (addToBreadCrumb) {
-            builder.remainder(child);
-          }
-          builder.withIndex();
-          builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
-          return builder.build();
-        } else {
-          logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
-          return null;
-        }
-      }
-    }
+    return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg);
   }
 
-  private MajorType getLastPathType() {
+  MajorType getLastPathType() {
     if((this.getField().getType().getMinorType() == MinorType.LIST  &&
         this.getField().getType().getMode() == DataMode.REPEATED)) {  // Use Repeated scalar type instead of Required List.
       VectorWithOrdinal vord = getChildVectorWithOrdinal(null);

http://git-wip-us.apache.org/repos/asf/drill/blob/bb69f220/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index 1706f5c..2c93c31 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -173,7 +173,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
    */
-  public void putChild(String name, ValueVector vector) {
+  protected void putChild(String name, ValueVector vector) {
     putVector(name, vector);
     field.addChild(vector.getField());
   }


Mime
View raw message