hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmccl...@apache.org
Subject [21/25] hive git commit: HIVE-14214: ORC Schema Evolution and Predicate Push Down do not work together (no rows returned) (Matt McCline, reviewed by Prasanth Jayachandran/Owen O'Malley)
Date Thu, 28 Jul 2016 18:53:45 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e4da9bf..28daed2 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -180,6 +180,10 @@ minitez.query.files.shared=acid_globallimit.q,\
   orc_merge_incompat1.q,\
   orc_merge_incompat2.q,\
   orc_merge_incompat3.q,\
+  orc_ppd_schema_evol_1a.q,\
+  orc_ppd_schema_evol_1b.q,\
+  orc_ppd_schema_evol_2a.q,\
+  orc_ppd_schema_evol_2b.q,\
   orc_vectorization_ppd.q,\
   parallel.q,\
   ptf.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/orc/src/java/org/apache/orc/impl/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RecordReaderImpl.java b/orc/src/java/org/apache/orc/impl/RecordReaderImpl.java
index 36a802e..f165be2 100644
--- a/orc/src/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/orc/src/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -76,6 +76,7 @@ public class RecordReaderImpl implements RecordReader {
   protected final TypeDescription schema;
   private final List<OrcProto.Type> types;
   private final int bufferSize;
+  private final SchemaEvolution evolution;
   private final boolean[] included;
   private final long rowIndexStride;
   private long rowInStripe = 0;
@@ -134,36 +135,50 @@ public class RecordReaderImpl implements RecordReader {
 
   protected RecordReaderImpl(ReaderImpl fileReader,
                              Reader.Options options) throws IOException {
-    SchemaEvolution treeReaderSchema;
     this.included = options.getInclude();
     included[0] = true;
     if (options.getSchema() == null) {
       if (LOG.isInfoEnabled()) {
-        LOG.info("Schema on read not provided -- using file schema " +
+        LOG.info("Reader schema not provided -- using file schema " +
             fileReader.getSchema());
       }
-      treeReaderSchema = new SchemaEvolution(fileReader.getSchema(), included);
+      evolution = new SchemaEvolution(fileReader.getSchema(), included);
     } else {
 
       // Now that we are creating a record reader for a file, validate that the schema to read
       // is compatible with the file schema.
       //
-      treeReaderSchema = new SchemaEvolution(fileReader.getSchema(),
+      evolution = new SchemaEvolution(fileReader.getSchema(),
           options.getSchema(),included);
+      if (LOG.isDebugEnabled() && evolution.hasConversion()) {
+        LOG.debug("ORC file " + fileReader.path.toString() +
+            " has data type conversion --\n" +
+            "reader schema: " + options.getSchema().toString() + "\n" +
+            "file schema:   " + fileReader.getSchema());
+      }
     }
-    this.schema = treeReaderSchema.getReaderSchema();
+    this.schema = evolution.getReaderSchema();
     this.path = fileReader.path;
     this.codec = fileReader.codec;
     this.types = fileReader.types;
     this.bufferSize = fileReader.bufferSize;
     this.rowIndexStride = fileReader.rowIndexStride;
     SearchArgument sarg = options.getSearchArgument();
-    if (sarg != null && rowIndexStride != 0) {
+    // We want to use the sarg for predicate evaluation but we have data type conversion
+    // (i.e Schema Evolution), so we currently ignore it.
+    if (sarg != null && rowIndexStride != 0 && !evolution.hasConversion()) {
       sargApp = new SargApplier(
           sarg, options.getColumnNames(), rowIndexStride, types,
           included.length);
     } else {
       sargApp = null;
+      if (evolution.hasConversion()) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(
+              "Skipping stripe elimination for {} since the schema has data type conversion",
+              fileReader.path);
+        }
+      }
     }
     long rows = 0;
     long skippedRows = 0;
@@ -205,8 +220,8 @@ public class RecordReaderImpl implements RecordReader {
       skipCorrupt = OrcConf.SKIP_CORRUPT_DATA.getBoolean(fileReader.conf);
     }
 
-    reader = TreeReaderFactory.createTreeReader(treeReaderSchema.getReaderSchema(),
-        treeReaderSchema, included, skipCorrupt);
+    reader = TreeReaderFactory.createTreeReader(evolution.getReaderSchema(),
+        evolution, included, skipCorrupt);
     indexes = new OrcProto.RowIndex[types.size()];
     bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()];
     advanceToNextRow(reader, 0L, true);

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/orc/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/SchemaEvolution.java b/orc/src/java/org/apache/orc/impl/SchemaEvolution.java
index 61a3f85..ce3af7a 100644
--- a/orc/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/orc/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -20,9 +20,8 @@ package org.apache.orc.impl;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -33,50 +32,64 @@ import org.apache.orc.TypeDescription;
  * has been schema evolution.
  */
 public class SchemaEvolution {
-  private final Map<Integer, TypeDescription> readerToFile;
+  private final TypeDescription[] readerFileTypes;
   private final boolean[] included;
   private final TypeDescription readerSchema;
+  private boolean hasConversion;
   private static final Log LOG = LogFactory.getLog(SchemaEvolution.class);
 
   public SchemaEvolution(TypeDescription readerSchema, boolean[] included) {
-    this.included = included;
-    readerToFile = null;
+    this.included = (included == null ? null : Arrays.copyOf(included, included.length));
     this.readerSchema = readerSchema;
+
+    hasConversion = false;
+
+    readerFileTypes = new TypeDescription[this.readerSchema.getMaximumId() + 1];
+    buildSameSchemaFileTypesArray();
   }
 
   public SchemaEvolution(TypeDescription fileSchema,
                          TypeDescription readerSchema,
                          boolean[] included) throws IOException {
-    readerToFile = new HashMap<>(readerSchema.getMaximumId() + 1);
-    this.included = included;
+    this.included = (included == null ? null : Arrays.copyOf(included, included.length));
     if (checkAcidSchema(fileSchema)) {
       this.readerSchema = createEventSchema(readerSchema);
     } else {
       this.readerSchema = readerSchema;
     }
-    buildMapping(fileSchema, this.readerSchema);
+
+    hasConversion = false;
+    readerFileTypes = new TypeDescription[this.readerSchema.getMaximumId() + 1];
+    buildConversionFileTypesArray(fileSchema, this.readerSchema);
   }
 
   public TypeDescription getReaderSchema() {
     return readerSchema;
   }
 
+  /**
+   * Is there Schema Evolution data type conversion?
+   * @return
+   */
+  public boolean hasConversion() {
+    return hasConversion;
+  }
+
   public TypeDescription getFileType(TypeDescription readerType) {
-    TypeDescription result;
-    if (readerToFile == null) {
-      if (included == null || included[readerType.getId()]) {
-        result = readerType;
-      } else {
-        result = null;
-      }
-    } else {
-      result = readerToFile.get(readerType.getId());
-    }
-    return result;
+    return getFileType(readerType.getId());
+  }
+
+  /**
+   * Get the file type by reader type id.
+   * @param readerType
+   * @return
+   */
+  public TypeDescription getFileType(int id) {
+    return readerFileTypes[id];
   }
 
-  void buildMapping(TypeDescription fileType,
-                    TypeDescription readerType) throws IOException {
+  void buildConversionFileTypesArray(TypeDescription fileType,
+                                     TypeDescription readerType) throws IOException {
     // if the column isn't included, don't map it
     if (included != null && !included[readerType.getId()]) {
       return;
@@ -101,9 +114,16 @@ public class SchemaEvolution {
         case CHAR:
         case VARCHAR:
           // We do conversion when same CHAR/VARCHAR type but different maxLength.
+          if (fileType.getMaxLength() != readerType.getMaxLength()) {
+            hasConversion = true;
+          }
           break;
         case DECIMAL:
           // We do conversion when same DECIMAL type but different precision/scale.
+          if (fileType.getPrecision() != readerType.getPrecision() ||
+              fileType.getScale() != readerType.getScale()) {
+            hasConversion = true;
+          }
           break;
         case UNION:
         case MAP:
@@ -113,7 +133,7 @@ public class SchemaEvolution {
           List<TypeDescription> readerChildren = readerType.getChildren();
           if (fileChildren.size() == readerChildren.size()) {
             for(int i=0; i < fileChildren.size(); ++i) {
-              buildMapping(fileChildren.get(i), readerChildren.get(i));
+              buildConversionFileTypesArray(fileChildren.get(i), readerChildren.get(i));
             }
           } else {
             isOk = false;
@@ -124,9 +144,12 @@ public class SchemaEvolution {
           // allow either side to have fewer fields than the other
           List<TypeDescription> fileChildren = fileType.getChildren();
           List<TypeDescription> readerChildren = readerType.getChildren();
+          if (fileChildren.size() != readerChildren.size()) {
+            hasConversion = true;
+          }
           int jointSize = Math.min(fileChildren.size(), readerChildren.size());
           for(int i=0; i < jointSize; ++i) {
-            buildMapping(fileChildren.get(i), readerChildren.get(i));
+            buildConversionFileTypesArray(fileChildren.get(i), readerChildren.get(i));
           }
           break;
         }
@@ -139,9 +162,14 @@ public class SchemaEvolution {
        */
 
       isOk = ConvertTreeReaderFactory.canConvert(fileType, readerType);
+      hasConversion = true;
     }
     if (isOk) {
-      readerToFile.put(readerType.getId(), fileType);
+      int id = readerType.getId();
+      if (readerFileTypes[id] != null) {
+        throw new RuntimeException("reader to file type entry already assigned");
+      }
+      readerFileTypes[id] = fileType;
     } else {
       throw new IOException(
           String.format(
@@ -151,6 +179,31 @@ public class SchemaEvolution {
     }
   }
 
+  /**
+   * Use to make a reader to file type array when the schema is the same.
+   * @return
+   */
+  private void buildSameSchemaFileTypesArray() {
+    buildSameSchemaFileTypesArrayRecurse(readerSchema);
+  }
+
+  void buildSameSchemaFileTypesArrayRecurse(TypeDescription readerType) {
+    if (included != null && !included[readerType.getId()]) {
+      return;
+    }
+    int id = readerType.getId();
+    if (readerFileTypes[id] != null) {
+      throw new RuntimeException("reader to file type entry already assigned");
+    }
+    readerFileTypes[id] = readerType;
+    List<TypeDescription> children = readerType.getChildren();
+    if (children != null) {
+      for (TypeDescription child : children) {
+        buildSameSchemaFileTypesArrayRecurse(child);
+      }
+    }
+  }
+
   private static boolean checkAcidSchema(TypeDescription type) {
     if (type.getCategory().equals(TypeDescription.Category.STRUCT)) {
       List<String> rootFields = type.getFieldNames();

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/orc/src/test/org/apache/orc/impl/TestSchemaEvolution.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/orc/src/test/org/apache/orc/impl/TestSchemaEvolution.java
new file mode 100644
index 0000000..3cd0390
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestSchemaEvolution.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.orc.TypeDescription;
+import org.junit.Test;
+
+public class TestSchemaEvolution {
+
+  @Test
+  public void testDataTypeConversion1() throws IOException {
+    TypeDescription fileStruct1 = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createInt())
+        .addField("f2", TypeDescription.createString())
+        .addField("f3", TypeDescription.createDecimal().withPrecision(38).withScale(10));
+    SchemaEvolution same1 = new SchemaEvolution(fileStruct1, null);
+    assertFalse(same1.hasConversion());
+    TypeDescription readerStruct1 = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createInt())
+        .addField("f2", TypeDescription.createString())
+        .addField("f3", TypeDescription.createDecimal().withPrecision(38).withScale(10));
+    SchemaEvolution both1 = new SchemaEvolution(fileStruct1, readerStruct1, null);
+    assertFalse(both1.hasConversion());
+    TypeDescription readerStruct1diff = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createLong())
+        .addField("f2", TypeDescription.createString())
+        .addField("f3", TypeDescription.createDecimal().withPrecision(38).withScale(10));
+    SchemaEvolution both1diff = new SchemaEvolution(fileStruct1, readerStruct1diff, null);
+    assertTrue(both1diff.hasConversion());
+    TypeDescription readerStruct1diffPrecision = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createInt())
+        .addField("f2", TypeDescription.createString())
+        .addField("f3", TypeDescription.createDecimal().withPrecision(12).withScale(10));
+    SchemaEvolution both1diffPrecision = new SchemaEvolution(fileStruct1, readerStruct1diffPrecision, null);
+    assertTrue(both1diffPrecision.hasConversion());
+  }
+
+  @Test
+  public void testDataTypeConversion2() throws IOException {
+    TypeDescription fileStruct2 = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createUnion()
+            .addUnionChild(TypeDescription.createByte())
+            .addUnionChild(TypeDescription.createDecimal()
+                .withPrecision(20).withScale(10)))
+        .addField("f2", TypeDescription.createStruct()
+            .addField("f3", TypeDescription.createDate())
+            .addField("f4", TypeDescription.createDouble())
+            .addField("f5", TypeDescription.createBoolean()))
+        .addField("f6", TypeDescription.createChar().withMaxLength(100));
+    SchemaEvolution same2 = new SchemaEvolution(fileStruct2, null);
+    assertFalse(same2.hasConversion());
+    TypeDescription readerStruct2 = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createUnion()
+            .addUnionChild(TypeDescription.createByte())
+            .addUnionChild(TypeDescription.createDecimal()
+                .withPrecision(20).withScale(10)))
+        .addField("f2", TypeDescription.createStruct()
+            .addField("f3", TypeDescription.createDate())
+            .addField("f4", TypeDescription.createDouble())
+            .addField("f5", TypeDescription.createBoolean()))
+        .addField("f6", TypeDescription.createChar().withMaxLength(100));
+    SchemaEvolution both2 = new SchemaEvolution(fileStruct2, readerStruct2, null);
+    assertFalse(both2.hasConversion());
+    TypeDescription readerStruct2diff = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createUnion()
+            .addUnionChild(TypeDescription.createByte())
+            .addUnionChild(TypeDescription.createDecimal()
+                .withPrecision(20).withScale(10)))
+        .addField("f2", TypeDescription.createStruct()
+            .addField("f3", TypeDescription.createDate())
+            .addField("f4", TypeDescription.createDouble()))
+        .addField("f6", TypeDescription.createChar().withMaxLength(100));
+    SchemaEvolution both2diff = new SchemaEvolution(fileStruct2, readerStruct2diff, null);
+    assertTrue(both2diff.hasConversion());
+    TypeDescription readerStruct2diffChar = TypeDescription.createStruct()
+        .addField("f1", TypeDescription.createUnion()
+            .addUnionChild(TypeDescription.createByte())
+            .addUnionChild(TypeDescription.createDecimal()
+                .withPrecision(20).withScale(10)))
+        .addField("f2", TypeDescription.createStruct()
+            .addField("f3", TypeDescription.createDate())
+            .addField("f4", TypeDescription.createDouble()))
+        .addField("f6", TypeDescription.createChar().withMaxLength(80));
+    SchemaEvolution both2diffChar = new SchemaEvolution(fileStruct2, readerStruct2diffChar, null);
+    assertTrue(both2diffChar.hasConversion());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index e9d77d5..e213a09 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.io.orc;
 
 import org.apache.orc.impl.InStream;
+import org.apache.orc.impl.SchemaEvolution;
 
   
 import java.io.IOException;
@@ -94,6 +95,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -521,16 +523,17 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private SplitStrategyKind splitStrategyKind;
     private final SearchArgument sarg;
 
-    Context(Configuration conf) {
+    Context(Configuration conf) throws IOException {
       this(conf, 1, null);
     }
 
-    Context(Configuration conf, final int minSplits) {
+    Context(Configuration conf, final int minSplits) throws IOException {
       this(conf, minSplits, null);
     }
 
     @VisibleForTesting
-    Context(Configuration conf, final int minSplits, ExternalFooterCachesByConf efc) {
+    Context(Configuration conf, final int minSplits, ExternalFooterCachesByConf efc)
+        throws IOException {
       this.conf = conf;
       this.forceThreadpool = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST);
       this.sarg = ConvertAstToSearchArg.createFromConf(conf);
@@ -575,6 +578,13 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
           // using such an aggregate fileId cache is not bulletproof and should be disable-able.
           boolean useExternalCache = HiveConf.getBoolVar(
               conf, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED);
+          if (useExternalCache) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(
+                "Turning off hive.orc.splits.ms.footer.cache.enabled since it is not fully supported yet");
+            }
+            useExternalCache = false;
+          }
           if (localCache == null) {
             localCache = new LocalCache(numThreads, cacheStripeDetailsSize, useSoftReference);
           }
@@ -651,19 +661,21 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final FileSystem fs;
     private final HdfsFileStatusWithId fileWithId;
     private final OrcTail orcTail;
+    private final List<OrcProto.Type> readerTypes;
     private final boolean isOriginal;
     private final List<DeltaMetaData> deltas;
     private final boolean hasBase;
     private final ByteBuffer ppdResult;
 
     SplitInfo(Context context, FileSystem fs, HdfsFileStatusWithId fileWithId, OrcTail orcTail,
-        boolean isOriginal, List<DeltaMetaData> deltas, boolean hasBase, Path dir,
-        boolean[] covered, ByteBuffer ppdResult) throws IOException {
+        List<OrcProto.Type> readerTypes, boolean isOriginal, List<DeltaMetaData> deltas,
+        boolean hasBase, Path dir, boolean[] covered, ByteBuffer ppdResult) throws IOException {
       super(dir, context.numBuckets, deltas, covered);
       this.context = context;
       this.fs = fs;
       this.fileWithId = fileWithId;
       this.orcTail = orcTail;
+      this.readerTypes = readerTypes;
       this.isOriginal = isOriginal;
       this.deltas = deltas;
       this.hasBase = hasBase;
@@ -672,10 +684,10 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
 
     @VisibleForTesting
     public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, OrcTail orcTail,
-        boolean isOriginal, ArrayList<DeltaMetaData> deltas, boolean hasBase, Path dir,
-        boolean[] covered) throws IOException {
+        List<OrcProto.Type> readerTypes,  boolean isOriginal, ArrayList<DeltaMetaData> deltas,
+        boolean hasBase, Path dir, boolean[] covered) throws IOException {
       this(context, fs, AcidUtils.createOriginalObj(null, fileStatus),
-          orcTail, isOriginal, deltas, hasBase, dir, covered, null);
+          orcTail, readerTypes, isOriginal, deltas, hasBase, dir, covered, null);
     }
   }
 
@@ -703,6 +715,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final List<DeltaMetaData> deltas;
     private final boolean[] covered;
     final boolean isOriginal;
+    final List<OrcProto.Type> readerTypes;
     // References to external fields for async SplitInfo generation.
     private List<Future<List<OrcSplit>>> splitFuturesRef = null;
     private List<OrcSplit> splitsRef = null;
@@ -710,13 +723,14 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final boolean allowSyntheticFileIds;
 
     public ETLSplitStrategy(Context context, FileSystem fs, Path dir,
-        List<HdfsFileStatusWithId> children, boolean isOriginal, List<DeltaMetaData> deltas,
-        boolean[] covered, UserGroupInformation ugi, boolean allowSyntheticFileIds) {
+        List<HdfsFileStatusWithId> children, List<OrcProto.Type> readerTypes, boolean isOriginal,
+        List<DeltaMetaData> deltas, boolean[] covered, UserGroupInformation ugi, boolean allowSyntheticFileIds) {
       assert !children.isEmpty();
       this.context = context;
       this.dirs = Lists.newArrayList(new ETLDir(dir, fs, children.size()));
       this.files = children;
       this.isOriginal = isOriginal;
+      this.readerTypes = readerTypes;
       this.deltas = deltas;
       this.covered = covered;
       this.ugi = ugi;
@@ -756,7 +770,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
           }
           // Ignore files eliminated by PPD, or of 0 length.
           if (ppdResult != FooterCache.NO_SPLIT_AFTER_PPD && file.getFileStatus().getLen() > 0) {
-            result.add(new SplitInfo(context, dir.fs, file, orcTail,
+            result.add(new SplitInfo(context, dir.fs, file, orcTail, readerTypes,
                 isOriginal, deltas, true, dir.dir, covered, ppdResult));
           }
         }
@@ -770,7 +784,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
           }
           // ignore files of 0 length
           if (file.getFileStatus().getLen() > 0) {
-            result.add(new SplitInfo(context, dir.fs, file, null,
+            result.add(new SplitInfo(context, dir.fs, file, null, readerTypes,
                 isOriginal, deltas, true, dir.dir, covered, null));
           }
         }
@@ -1057,10 +1071,11 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final long blockSize;
     private final TreeMap<Long, BlockLocation> locations;
     private OrcTail orcTail;
+    private final List<OrcProto.Type> readerTypes;
     private List<StripeInformation> stripes;
     private List<StripeStatistics> stripeStats;
-    private List<OrcProto.Type> types;
-    private boolean[] includedCols;
+    private List<OrcProto.Type> fileTypes;
+    private boolean[] readerIncluded;
     private final boolean isOriginal;
     private final List<DeltaMetaData> deltas;
     private final boolean hasBase;
@@ -1070,6 +1085,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     private final ByteBuffer ppdResult;
     private final UserGroupInformation ugi;
     private final boolean allowSyntheticFileIds;
+    private SchemaEvolution evolution;
 
     public SplitGenerator(SplitInfo splitInfo, UserGroupInformation ugi,
         boolean allowSyntheticFileIds) throws IOException {
@@ -1080,6 +1096,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
       this.fsFileId = splitInfo.fileWithId.getFileId();
       this.blockSize = this.file.getBlockSize();
       this.orcTail = splitInfo.orcTail;
+      this.readerTypes = splitInfo.readerTypes;
       // TODO: potential DFS call
       this.locations = SHIMS.getLocationsWithOffset(fs, file);
       this.isOriginal = splitInfo.isOriginal;
@@ -1251,12 +1268,24 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
         // We can't eliminate stripes if there are deltas because the
         // deltas may change the rows making them match the predicate.
         if ((deltas == null || deltas.isEmpty()) && context.sarg != null) {
-          String[] colNames = extractNeededColNames(types, context.conf, includedCols, isOriginal);
-          if (colNames == null) {
-            LOG.warn("Skipping split elimination for {} as column names is null", file.getPath());
+          // Also, we currently do not use predicate evaluation when the schema has data type
+          // conversion.
+          if (evolution.hasConversion()) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(
+                  "Skipping split elimination for {} since the schema has data type conversion",
+                  file.getPath());
+            }
           } else {
-            includeStripe = pickStripes(context.sarg, colNames, writerVersion, isOriginal,
-                stripeStats, stripes.size(), file.getPath());
+            String[] colNames =
+                extractNeededColNames((readerTypes == null ? fileTypes : readerTypes),
+                    context.conf, readerIncluded, isOriginal);
+            if (colNames == null) {
+              LOG.warn("Skipping split elimination for {} as column names is null", file.getPath());
+            } else {
+              includeStripe = pickStripes(context.sarg, colNames, writerVersion, isOriginal,
+                  stripeStats, stripes.size(), file.getPath());
+            }
           }
         }
         return generateSplitsFromStripes(includeStripe);
@@ -1370,29 +1399,67 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
       }
       stripes = orcTail.getStripes();
       stripeStats = orcTail.getStripeStatistics();
-      types = orcTail.getTypes();
+      fileTypes = orcTail.getTypes();
+      TypeDescription fileSchema = OrcUtils.convertTypeFromProtobuf(fileTypes, 0);
+      if (readerTypes == null) {
+        readerIncluded = genIncludedColumns(fileTypes, context.conf, isOriginal);
+        evolution = new SchemaEvolution(fileSchema, readerIncluded);
+      } else {
+        // The readerSchema always comes in without ACID columns.
+        readerIncluded = genIncludedColumns(readerTypes, context.conf, /* isOriginal */ true);
+        if (readerIncluded != null && !isOriginal) {
+          // We shift the include columns here because the SchemaEvolution constructor will
+          // add the ACID event metadata the readerSchema...
+          readerIncluded = shiftReaderIncludedForAcid(readerIncluded);
+        }
+        TypeDescription readerSchema = OrcUtils.convertTypeFromProtobuf(readerTypes, 0);
+        evolution = new SchemaEvolution(fileSchema, readerSchema, readerIncluded);
+      }
       writerVersion = orcTail.getWriterVersion();
-      includedCols = genIncludedColumns(types, context.conf, isOriginal);
       List<OrcProto.ColumnStatistics> fileColStats = orcTail.getFooter().getStatisticsList();
-      projColsUncompressedSize = computeProjectionSize(types, fileColStats, includedCols,
+      boolean[] fileIncluded;
+      if (readerTypes == null) {
+        fileIncluded = readerIncluded;
+      } else {
+        fileIncluded = new boolean[fileTypes.size()];
+        final int readerSchemaSize = readerTypes.size();
+        for (int i = 0; i < readerSchemaSize; i++) {
+          TypeDescription fileType = evolution.getFileType(i);
+          if (fileType != null) {
+            fileIncluded[fileType.getId()] = true;
+          }
+        }
+      }
+      projColsUncompressedSize = computeProjectionSize(fileTypes, fileColStats, fileIncluded,
           isOriginal);
       if (!context.footerInSplits) {
         orcTail = null;
       }
     }
 
-    private long computeProjectionSize(List<OrcProto.Type> types,
-        List<OrcProto.ColumnStatistics> stats, boolean[] includedCols, boolean isOriginal) {
+    private long computeProjectionSize(List<OrcProto.Type> fileTypes,
+        List<OrcProto.ColumnStatistics> stats, boolean[] fileIncluded, boolean isOriginal) {
       final int rootIdx = getRootColumn(isOriginal);
       List<Integer> internalColIds = Lists.newArrayList();
-      if (includedCols != null) {
-        for (int i = 0; i < includedCols.length; i++) {
-          if (includedCols[i]) {
+      if (fileIncluded != null) {
+        for (int i = 0; i < fileIncluded.length; i++) {
+          if (fileIncluded[i]) {
             internalColIds.add(rootIdx + i);
           }
         }
       }
-      return ReaderImpl.getRawDataSizeFromColIndices(internalColIds, types, stats);
+      return ReaderImpl.getRawDataSizeFromColIndices(internalColIds, fileTypes, stats);
+    }
+
+    private boolean[] shiftReaderIncludedForAcid(boolean[] included) {
+      // We always need the base row
+      included[0] = true;
+      boolean[] newIncluded = new boolean[included.length + OrcRecordUpdater.FIELDS];
+      Arrays.fill(newIncluded, 0, OrcRecordUpdater.FIELDS, true);
+      for(int i= 0; i < included.length; ++i) {
+        newIncluded[i + OrcRecordUpdater.FIELDS] = included[i];
+      }
+      return newIncluded;
     }
   }
 
@@ -1427,6 +1494,21 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
       pathFutures.add(ecs.submit(fileGenerator));
     }
 
+    boolean isTransactionalTableScan =
+        HiveConf.getBoolVar(conf, ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN);
+    boolean isSchemaEvolution = HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION);
+    TypeDescription readerSchema =
+        OrcInputFormat.getDesiredRowTypeDescr(conf, isTransactionalTableScan, Integer.MAX_VALUE);
+    List<OrcProto.Type> readerTypes = null;
+    if (readerSchema != null) {
+      readerTypes = OrcUtils.getOrcTypes(readerSchema);
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Generate splits schema evolution property " + isSchemaEvolution +
+        " reader schema " + (readerSchema == null ? "NULL" : readerSchema.toString()) +
+        " transactional scan property " + isTransactionalTableScan);
+    }
+
     // complete path futures and schedule split generation
     try {
       CombinedCtx combinedCtx = (context.splitStrategyBatchMs > 0) ? new CombinedCtx() : null;
@@ -1455,7 +1537,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
         // We have received a new directory information, make a split strategy.
         --resultsLeft;
         SplitStrategy<?> splitStrategy = determineSplitStrategy(combinedCtx, context, adi.fs,
-            adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, ugi, allowSyntheticFileIds);
+            adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, readerTypes, ugi,
+            allowSyntheticFileIds);
         if (splitStrategy == null) continue; // Combined.
 
         if (isDebugEnabled) {
@@ -1512,7 +1595,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
   @VisibleForTesting
   // We could have this as a protected method w/no class, but half of Hive is static, so there.
   public static class ContextFactory {
-    public Context create(Configuration conf, int numSplits) {
+    public Context create(Configuration conf, int numSplits) throws IOException {
       return new Context(conf, numSplits);
     }
   }
@@ -1533,14 +1616,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
 
   private static SplitStrategy<?> combineOrCreateETLStrategy(CombinedCtx combinedCtx,
       Context context, FileSystem fs, Path dir, List<HdfsFileStatusWithId> files,
-      List<DeltaMetaData> deltas, boolean[] covered, boolean isOriginal,
-      UserGroupInformation ugi, boolean allowSyntheticFileIds) {
+      List<DeltaMetaData> deltas, boolean[] covered, List<OrcProto.Type> readerTypes,
+      boolean isOriginal, UserGroupInformation ugi, boolean allowSyntheticFileIds) {
     if (!deltas.isEmpty() || combinedCtx == null) {
       return new ETLSplitStrategy(
-          context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds);
+          context, fs, dir, files, readerTypes, isOriginal, deltas, covered, ugi,
+          allowSyntheticFileIds);
     } else if (combinedCtx.combined == null) {
       combinedCtx.combined = new ETLSplitStrategy(
-          context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds);
+          context, fs, dir, files, readerTypes, isOriginal, deltas, covered, ugi,
+          allowSyntheticFileIds);
       combinedCtx.combineStartUs = System.nanoTime();
       return null;
     } else {
@@ -1550,11 +1635,13 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
       case YES: return null;
       case NO_AND_CONTINUE:
         return new ETLSplitStrategy(
-            context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds);
+            context, fs, dir, files, readerTypes, isOriginal, deltas, covered, ugi,
+            allowSyntheticFileIds);
       case NO_AND_SWAP: {
         ETLSplitStrategy oldBase = combinedCtx.combined;
         combinedCtx.combined = new ETLSplitStrategy(
-            context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds);
+            context, fs, dir, files, readerTypes, isOriginal, deltas, covered, ugi,
+            allowSyntheticFileIds);
         combinedCtx.combineStartUs = System.nanoTime();
         return oldBase;
       }
@@ -1864,8 +1951,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
   @VisibleForTesting
   static SplitStrategy<?> determineSplitStrategy(CombinedCtx combinedCtx, Context context,
       FileSystem fs, Path dir, AcidUtils.Directory dirInfo,
-      List<HdfsFileStatusWithId> baseOrOriginalFiles, UserGroupInformation ugi,
-      boolean allowSyntheticFileIds) {
+      List<HdfsFileStatusWithId> baseOrOriginalFiles, List<OrcProto.Type> readerTypes,
+      UserGroupInformation ugi, boolean allowSyntheticFileIds) {
     Path base = dirInfo.getBaseDirectory();
     List<HdfsFileStatusWithId> original = dirInfo.getOriginalFiles();
     List<DeltaMetaData> deltas = AcidUtils.serializeDeltas(dirInfo.getCurrentDirectories());
@@ -1898,12 +1985,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
         case ETL:
           // ETL strategy requested through config
           return combineOrCreateETLStrategy(combinedCtx, context, fs, dir, baseOrOriginalFiles,
-              deltas, covered, isOriginal, ugi, allowSyntheticFileIds);
+              deltas, covered, readerTypes, isOriginal, ugi, allowSyntheticFileIds);
         default:
           // HYBRID strategy
           if (avgFileSize > context.maxSize || totalFiles <= context.etlFileThreshold) {
             return combineOrCreateETLStrategy(combinedCtx, context, fs, dir, baseOrOriginalFiles,
-                deltas, covered, isOriginal, ugi, allowSyntheticFileIds);
+                deltas, covered, readerTypes, isOriginal, ugi, allowSyntheticFileIds);
           } else {
             return new BISplitStrategy(context, fs, dir, baseOrOriginalFiles,
                 isOriginal, deltas, covered, allowSyntheticFileIds);

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
index c4a7226..3b05fb2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
@@ -136,7 +136,7 @@ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{
   }
 
   // Nearly C/P from OrcInputFormat; there are too many statics everywhere to sort this out.
-  private Context createContext(Configuration conf, int numSplits) {
+  private Context createContext(Configuration conf, int numSplits) throws IOException {
     // Use threads to resolve directories into splits.
     if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED)) {
       // Create HiveConf once, since this is expensive.

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 3c89dd6..cddd8d5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -768,7 +768,7 @@ public class TestInputOutputFormat {
       MockFileSystem fs, String path, OrcInputFormat.CombinedCtx combineCtx) throws IOException {
     OrcInputFormat.AcidDirInfo adi = createAdi(context, fs, path);
     return OrcInputFormat.determineSplitStrategy(combineCtx, context,
-        adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, true);
+        adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, null, true);
   }
 
   public OrcInputFormat.AcidDirInfo createAdi(
@@ -781,7 +781,7 @@ public class TestInputOutputFormat {
       OrcInputFormat.Context context, OrcInputFormat.FileGenerator gen) throws IOException {
     OrcInputFormat.AcidDirInfo adi = gen.call();
     return OrcInputFormat.determineSplitStrategy(
-        null, context, adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, true);
+        null, context, adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, null, true);
   }
 
   public static class MockBlock {
@@ -1389,7 +1389,7 @@ public class TestInputOutputFormat {
     OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
     OrcInputFormat.SplitGenerator splitter =
         new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-            fs.getFileStatus(new Path("/a/file")), null, true,
+            fs.getFileStatus(new Path("/a/file")), null, null, true,
             new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null), null, true);
     OrcSplit result = splitter.createSplit(0, 200, null);
     assertEquals(0, result.getStart());
@@ -1430,7 +1430,7 @@ public class TestInputOutputFormat {
     OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
     OrcInputFormat.SplitGenerator splitter =
         new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-            fs.getFileStatus(new Path("/a/file")), null, true,
+            fs.getFileStatus(new Path("/a/file")), null, null, true,
             new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null), null, true);
     List<OrcSplit> results = splitter.call();
     OrcSplit result = results.get(0);
@@ -1453,7 +1453,7 @@ public class TestInputOutputFormat {
     HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0);
     context = new OrcInputFormat.Context(conf);
     splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-      fs.getFileStatus(new Path("/a/file")), null, true,
+      fs.getFileStatus(new Path("/a/file")), null, null, true,
         new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null), null, true);
     results = splitter.call();
     for(int i=0; i < stripeSizes.length; ++i) {
@@ -1481,7 +1481,7 @@ public class TestInputOutputFormat {
     OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
     OrcInputFormat.SplitGenerator splitter =
         new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-            fs.getFileStatus(new Path("/a/file")), null, true,
+            fs.getFileStatus(new Path("/a/file")), null, null, true,
             new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null), null, true);
     List<OrcSplit> results = splitter.call();
     OrcSplit result = results.get(0);
@@ -1503,7 +1503,7 @@ public class TestInputOutputFormat {
     HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0);
     context = new OrcInputFormat.Context(conf);
     splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-        fs.getFileStatus(new Path("/a/file")), null, true,
+        fs.getFileStatus(new Path("/a/file")), null, null, true,
         new ArrayList<AcidInputFormat.DeltaMetaData>(),
         true, null, null), null, true);
     results = splitter.call();
@@ -1523,7 +1523,7 @@ public class TestInputOutputFormat {
     HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 100000);
     context = new OrcInputFormat.Context(conf);
     splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
-        fs.getFileStatus(new Path("/a/file")), null, true,
+        fs.getFileStatus(new Path("/a/file")), null, null, true,
         new ArrayList<AcidInputFormat.DeltaMetaData>(),
         true, null, null), null, true);
     results = splitter.call();

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
index 62a0ab0..b26401d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -536,11 +537,13 @@ public class TestOrcSplitElimination {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestOrcSplitElimination.class);
 
+  @Ignore("External cache has been turned off for now")
   @Test
   public void testExternalFooterCache() throws Exception {
     testFooterExternalCacheImpl(false);
   }
 
+  @Ignore("External cache has been turned off for now")
   @Test
   public void testExternalFooterCachePpd() throws Exception {
     testFooterExternalCacheImpl(true);

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1a.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1a.q b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1a.q
new file mode 100644
index 0000000..f52c2ac
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1a.q
@@ -0,0 +1,42 @@
+set hive.cli.print.header=true;
+set hive.metastore.disallow.incompatible.col.type.changes=true;
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+
+create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_1.txt' into table unique_1;
+
+create table test1 stored as orc as select * from unique_1;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+
+alter table test1 change column i i string;
+
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test1 where i = '-1591211872';
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test1 where i = -1591211872;
+
+set hive.optimize.ppd=true;
+set hive.optimize.index.filter=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test1 where i = '-1591211872';
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test1 where i = -1591211872;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1b.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1b.q b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1b.q
new file mode 100644
index 0000000..e669f06
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_1b.q
@@ -0,0 +1,60 @@
+set hive.cli.print.header=true;
+set hive.metastore.disallow.incompatible.col.type.changes=true;
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_1.txt' into table unique_1;
+
+create table unique_2( 
+i int, 
+d double, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_2.txt' into table unique_2;
+
+create table test_two_files( 
+i int, 
+d double, 
+s string)
+stored as orc;
+
+insert into table test_two_files select * from unique_1 where i <= 0;
+insert into table test_two_files select * from unique_2 where i > 0;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+
+alter table test_two_files change column i i string;
+
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test_two_files where i = -1591211872;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test_two_files where i = -1591211872;
+
+
+set hive.optimize.ppd=true;
+set hive.optimize.index.filter=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test_two_files where i = '-1591211872';
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test_two_files where i = -1591211872;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2a.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2a.q b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2a.q
new file mode 100644
index 0000000..609924d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2a.q
@@ -0,0 +1,43 @@
+set hive.cli.print.header=true;
+set hive.metastore.disallow.incompatible.col.type.changes=false;
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+
+
+create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_1.txt' into table unique_1;
+
+create table test1 stored as orc as select * from unique_1 order by d;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+
+alter table test1 change column d d double;
+
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test1 where d = -4996703.42;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test1 where d = -4996703.42;
+
+set hive.optimize.ppd=true;
+set hive.optimize.index.filter=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test1 where d = -4996703.42;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test1 where d = -4996703.42;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2b.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2b.q b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2b.q
new file mode 100644
index 0000000..edebeef
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_ppd_schema_evol_2b.q
@@ -0,0 +1,62 @@
+set hive.cli.print.header=true;
+set hive.metastore.disallow.incompatible.col.type.changes=false;
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+
+create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_1.txt' into table unique_1;
+
+create table unique_2( 
+i int, 
+d string, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile;
+
+load data local inpath '../../data/files/unique_2.txt' into table unique_2;
+
+create table test_two_files( 
+i int, 
+d string, 
+s string)
+stored as orc;
+
+insert into table test_two_files select * from unique_1 where cast(d as double) <= 0 order by cast(d as double);
+insert into table test_two_files select * from unique_2 where cast(d as double) > 0 order by cast(d as double);
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
+
+alter table test_two_files change column d d double;
+
+set hive.optimize.ppd=false;
+set hive.optimize.index.filter=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test_two_files where d = -4996703.42;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test_two_files where d = -4996703.42;
+
+
+set hive.optimize.ppd=true;
+set hive.optimize.index.filter=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+select s from test_two_files where d = -4996703.42;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select s from test_two_files where d = -4996703.42;
+
+

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1a.q.out b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1a.q.out
new file mode 100644
index 0000000..5cefb3f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1a.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table test1 stored as orc as select * from unique_1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: create table test1 stored as orc as select * from unique_1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test1.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test1.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: alter table test1 change column i i string
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test1
+PREHOOK: Output: default@test1
+PREHOOK: query: select s from test1 where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1b.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1b.q.out b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1b.q.out
new file mode 100644
index 0000000..7fa580c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_1b.q.out
@@ -0,0 +1,124 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table unique_2( 
+i int, 
+d double, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: create table unique_2( 
+i int, 
+d double, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: create table test_two_files( 
+i int, 
+d double, 
+s string)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: create table test_two_files( 
+i int, 
+d double, 
+s string)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_two_files
+PREHOOK: query: insert into table test_two_files select * from unique_1 where i <= 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_1 where i <= 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: insert into table test_two_files select * from unique_2 where i > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_2
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_2 where i > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_2
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_2)unique_2.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_2)unique_2.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_2)unique_2.FieldSchema(name:s, type:string, comment:null), ]
+unique_2.i	unique_2.d	unique_2.s
+PREHOOK: query: alter table test_two_files change column i i string
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test_two_files
+PREHOOK: Output: default@test_two_files
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2a.q.out b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2a.q.out
new file mode 100644
index 0000000..5dead1c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2a.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table test1 stored as orc as select * from unique_1 order by d
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: create table test1 stored as orc as select * from unique_1 order by d
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test1.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test1.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: alter table test1 change column d d double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test1
+PREHOOK: Output: default@test1
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2b.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2b.q.out b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2b.q.out
new file mode 100644
index 0000000..b85ec98
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_ppd_schema_evol_2b.q.out
@@ -0,0 +1,124 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table unique_2( 
+i int, 
+d string, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: create table unique_2( 
+i int, 
+d string, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: create table test_two_files( 
+i int, 
+d string, 
+s string)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: create table test_two_files( 
+i int, 
+d string, 
+s string)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_two_files
+PREHOOK: query: insert into table test_two_files select * from unique_1 where cast(d as double) <= 0 order by cast(d as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_1 where cast(d as double) <= 0 order by cast(d as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: insert into table test_two_files select * from unique_2 where cast(d as double) > 0 order by cast(d as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_2
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_2 where cast(d as double) > 0 order by cast(d as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_2
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_2)unique_2.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_2)unique_2.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_2)unique_2.FieldSchema(name:s, type:string, comment:null), ]
+unique_2.i	unique_2.d	unique_2.s
+PREHOOK: query: alter table test_two_files change column d d double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test_two_files
+PREHOOK: Output: default@test_two_files
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1a.q.out b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1a.q.out
new file mode 100644
index 0000000..5cefb3f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1a.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table test1 stored as orc as select * from unique_1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: create table test1 stored as orc as select * from unique_1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test1.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test1.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: alter table test1 change column i i string
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test1
+PREHOOK: Output: default@test1
+PREHOOK: query: select s from test1 where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test1 where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+fred king

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1b.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1b.q.out b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1b.q.out
new file mode 100644
index 0000000..7fa580c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_1b.q.out
@@ -0,0 +1,124 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d double, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table unique_2( 
+i int, 
+d double, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: create table unique_2( 
+i int, 
+d double, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: create table test_two_files( 
+i int, 
+d double, 
+s string)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: create table test_two_files( 
+i int, 
+d double, 
+s string)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_two_files
+PREHOOK: query: insert into table test_two_files select * from unique_1 where i <= 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_1 where i <= 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: insert into table test_two_files select * from unique_2 where i > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_2
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_2 where i > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_2
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_2)unique_2.FieldSchema(name:d, type:double, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_2)unique_2.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_2)unique_2.FieldSchema(name:s, type:string, comment:null), ]
+unique_2.i	unique_2.d	unique_2.s
+PREHOOK: query: alter table test_two_files change column i i string
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test_two_files
+PREHOOK: Output: default@test_two_files
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = '-1591211872'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king
+PREHOOK: query: select s from test_two_files where i = -1591211872
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+fred king

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2a.q.out b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2a.q.out
new file mode 100644
index 0000000..5dead1c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2a.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table test1 stored as orc as select * from unique_1 order by d
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: create table test1 stored as orc as select * from unique_1 order by d
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test1.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test1.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: alter table test1 change column d d double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test1
+PREHOOK: Output: default@test1
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test1 where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+#### A masked pattern was here ####
+s
+luke quirinius

http://git-wip-us.apache.org/repos/asf/hive/blob/05784725/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2b.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2b.q.out b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2b.q.out
new file mode 100644
index 0000000..b85ec98
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_ppd_schema_evol_2b.q.out
@@ -0,0 +1,124 @@
+PREHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: create table unique_1( 
+i int, 
+d string, 
+s string) 
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_1
+POSTHOOK: query: load data local inpath '../../data/files/unique_1.txt' into table unique_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_1
+PREHOOK: query: create table unique_2( 
+i int, 
+d string, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: create table unique_2( 
+i int, 
+d string, 
+s string)
+row format delimited 
+fields terminated by '|' 
+stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_2
+POSTHOOK: query: load data local inpath '../../data/files/unique_2.txt' into table unique_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_2
+PREHOOK: query: create table test_two_files( 
+i int, 
+d string, 
+s string)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: create table test_two_files( 
+i int, 
+d string, 
+s string)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_two_files
+PREHOOK: query: insert into table test_two_files select * from unique_1 where cast(d as double) <= 0 order by cast(d as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_1
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_1 where cast(d as double) <= 0 order by cast(d as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_1
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_1)unique_1.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_1)unique_1.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_1)unique_1.FieldSchema(name:s, type:string, comment:null), ]
+unique_1.i	unique_1.d	unique_1.s
+PREHOOK: query: insert into table test_two_files select * from unique_2 where cast(d as double) > 0 order by cast(d as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_2
+PREHOOK: Output: default@test_two_files
+POSTHOOK: query: insert into table test_two_files select * from unique_2 where cast(d as double) > 0 order by cast(d as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_2
+POSTHOOK: Output: default@test_two_files
+POSTHOOK: Lineage: test_two_files.d SIMPLE [(unique_2)unique_2.FieldSchema(name:d, type:string, comment:null), ]
+POSTHOOK: Lineage: test_two_files.i SIMPLE [(unique_2)unique_2.FieldSchema(name:i, type:int, comment:null), ]
+POSTHOOK: Lineage: test_two_files.s SIMPLE [(unique_2)unique_2.FieldSchema(name:s, type:string, comment:null), ]
+unique_2.i	unique_2.d	unique_2.s
+PREHOOK: query: alter table test_two_files change column d d double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@test_two_files
+PREHOOK: Output: default@test_two_files
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius
+PREHOOK: query: select s from test_two_files where d = -4996703.42
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_two_files
+#### A masked pattern was here ####
+s
+luke quirinius


Mime
View raw message