drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [3/5] drill git commit: DRILL-4980: Upgrading of the approach of parquet date correctness status detection - Parquet writer version is added; - Updated the detection method of parquet date correctness.
Date Wed, 23 Nov 2016 23:42:11 GMT
DRILL-4980: Upgrading of the approach of parquet date correctness status detection - Parquet
writer version is added; - Updated the detection method of parquet date correctness.

This closes #644


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cf29b917
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cf29b917
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cf29b917

Branch: refs/heads/master
Commit: cf29b917af4002a1ba0ef750ff3f5a553e910f8e
Parents: 2de3a27
Author: Vitalii Diravka <vitalii.diravka@gmail.com>
Authored: Wed Oct 26 12:22:06 2016 +0000
Committer: Parth Chandra <parthc@apache.org>
Committed: Wed Nov 23 09:33:29 2016 -0800

----------------------------------------------------------------------
 .../drill/exec/store/parquet/Metadata.java      |  61 ++++++++-----------
 .../store/parquet/ParquetReaderUtility.java     |  48 ++++++++++-----
 .../exec/store/parquet/ParquetRecordWriter.java |   4 +-
 .../drill/exec/store/parquet/ParquetWriter.java |  12 ++++
 .../TestCorruptParquetDateCorrection.java       |   6 +-
 .../0_0_1.parquet                               | Bin 290 -> 292 bytes
 .../0_0_2.parquet                               | Bin 290 -> 292 bytes
 .../0_0_3.parquet                               | Bin 290 -> 292 bytes
 .../0_0_4.parquet                               | Bin 290 -> 292 bytes
 .../0_0_5.parquet                               | Bin 290 -> 292 bytes
 .../0_0_6.parquet                               | Bin 290 -> 292 bytes
 .../4203_corrected_dates.parquet                | Bin 311 -> 313 bytes
 ...on_partitioned_metadata.requires_replace.txt |   2 +-
 13 files changed, 76 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index 04a2476..7b3afb4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -189,9 +189,9 @@ public class Metadata {
         childFiles.add(file);
       }
     }
-    ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(true);
+    ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(DrillVersionInfo.getVersion());
     if (childFiles.size() > 0) {
-      List<ParquetFileMetadata_v3> childFilesMetadata =
+      List<ParquetFileMetadata_v3 > childFilesMetadata =
           getParquetFileMetadata_v3(parquetTableMetadata, childFiles);
       metaDataList.addAll(childFilesMetadata);
       // Note that we do not need to merge the columnInfo at this point. The columnInfo is
already added
@@ -674,9 +674,6 @@ public class Metadata {
     @JsonIgnore public abstract ParquetTableMetadataBase clone();
 
     @JsonIgnore public abstract String getDrillVersion();
-
-    @JsonIgnore public abstract boolean isDateCorrect();
-
   }
 
   public static abstract class ParquetFileMetadata {
@@ -811,12 +808,6 @@ public class Metadata {
     public String getDrillVersion() {
       return null;
     }
-
-    @JsonIgnore @Override
-    public boolean isDateCorrect() {
-      return false;
-    }
-
   }
 
 
@@ -1025,31 +1016,29 @@ public class Metadata {
     @JsonProperty List<ParquetFileMetadata_v2> files;
     @JsonProperty List<String> directories;
     @JsonProperty String drillVersion;
-    @JsonProperty boolean isDateCorrect;
 
     public ParquetTableMetadata_v2() {
       super();
     }
 
-    public ParquetTableMetadata_v2(boolean isDateCorrect) {
-      this.drillVersion = DrillVersionInfo.getVersion();
-      this.isDateCorrect = isDateCorrect;
+    public ParquetTableMetadata_v2(String drillVersion) {
+      this.drillVersion = drillVersion;
     }
 
     public ParquetTableMetadata_v2(ParquetTableMetadataBase parquetTable,
-        List<ParquetFileMetadata_v2> files, List<String> directories) {
+        List<ParquetFileMetadata_v2> files, List<String> directories, String
drillVersion) {
       this.files = files;
       this.directories = directories;
       this.columnTypeInfo = ((ParquetTableMetadata_v2) parquetTable).columnTypeInfo;
-      this.drillVersion = DrillVersionInfo.getVersion();
-      this.isDateCorrect = true;
+      this.drillVersion = drillVersion;
     }
 
     public ParquetTableMetadata_v2(List<ParquetFileMetadata_v2> files, List<String>
directories,
-        ConcurrentHashMap<ColumnTypeMetadata_v2.Key, ColumnTypeMetadata_v2> columnTypeInfo)
{
+        ConcurrentHashMap<ColumnTypeMetadata_v2.Key, ColumnTypeMetadata_v2> columnTypeInfo,
String drillVersion) {
       this.files = files;
       this.directories = directories;
       this.columnTypeInfo = columnTypeInfo;
+      this.drillVersion = drillVersion;
     }
 
     public ColumnTypeMetadata_v2 getColumnTypeInfo(String[] name) {
@@ -1096,7 +1085,7 @@ public class Metadata {
     }
 
     @JsonIgnore @Override public ParquetTableMetadataBase clone() {
-      return new ParquetTableMetadata_v2(files, directories, columnTypeInfo);
+      return new ParquetTableMetadata_v2(files, directories, columnTypeInfo, drillVersion);
     }
 
     @JsonIgnore @Override
@@ -1104,10 +1093,6 @@ public class Metadata {
       return drillVersion;
     }
 
-    @JsonIgnore @Override public boolean isDateCorrect() {
-      return isDateCorrect;
-    }
-
   }
 
 
@@ -1383,30 +1368,38 @@ public class Metadata {
     @JsonProperty List<ParquetFileMetadata_v3> files;
     @JsonProperty List<String> directories;
     @JsonProperty String drillVersion;
-    @JsonProperty boolean isDateCorrect;
 
+    /**
+     * Default constructor needed for deserialization from Parquet Metadata Cache Files
+     * or for creating an empty instances of this class for the case when the Metadata Cache
File is absent
+     */
     public ParquetTableMetadata_v3() {
       super();
     }
 
-    public ParquetTableMetadata_v3(boolean isDateCorrect) {
-      this.drillVersion = DrillVersionInfo.getVersion();
-      this.isDateCorrect = isDateCorrect;
+    /**
+     * Used for creating the Parquet Metadata Cache File
+     * @param drillVersion  actual version of apache drill
+     */
+    public ParquetTableMetadata_v3(String drillVersion) {
+      this.drillVersion = drillVersion;
     }
 
     public ParquetTableMetadata_v3(ParquetTableMetadataBase parquetTable,
-        List<ParquetFileMetadata_v3> files, List<String> directories) {
+        List<ParquetFileMetadata_v3> files, List<String> directories, String
drillVersion) {
       this.files = files;
       this.directories = directories;
       this.columnTypeInfo = ((ParquetTableMetadata_v3) parquetTable).columnTypeInfo;
-      this.isDateCorrect = true;
+      this.drillVersion = drillVersion;
     }
 
     public ParquetTableMetadata_v3(List<ParquetFileMetadata_v3> files, List<String>
directories,
-        ConcurrentHashMap<ColumnTypeMetadata_v3.Key, ColumnTypeMetadata_v3> columnTypeInfo)
{
+        ConcurrentHashMap<ColumnTypeMetadata_v3.Key, ColumnTypeMetadata_v3> columnTypeInfo,
+        String drillVersion) {
       this.files = files;
       this.directories = directories;
       this.columnTypeInfo = columnTypeInfo;
+      this.drillVersion = drillVersion;
     }
 
     public ColumnTypeMetadata_v3 getColumnTypeInfo(String[] name) {
@@ -1453,7 +1446,7 @@ public class Metadata {
     }
 
     @JsonIgnore @Override public ParquetTableMetadataBase clone() {
-      return new ParquetTableMetadata_v3(files, directories, columnTypeInfo);
+      return new ParquetTableMetadata_v3(files, directories, columnTypeInfo, drillVersion);
     }
 
     @JsonIgnore @Override
@@ -1461,10 +1454,6 @@ public class Metadata {
       return drillVersion;
     }
 
-    @JsonIgnore @Override public boolean isDateCorrect() {
-      return isDateCorrect;
-    }
-
   }
 
 

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
index 767c98d..b22e666 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
@@ -59,19 +59,26 @@ public class ParquetReaderUtility {
    */
   public static final long JULIAN_DAY_NUMBER_FOR_UNIX_EPOCH = 2440588;
   /**
-   * All old parquet files (which haven't "is.date.correct=true" property in metadata) have
-   * a corrupt date shift: {@value} days or 2 * {@value #JULIAN_DAY_NUMBER_FOR_UNIX_EPOCH}
+   * All old parquet files (which haven't "is.date.correct=true" or "parquet-writer.version"
properties
+   * in metadata) have a corrupt date shift: {@value} days or 2 * {@value #JULIAN_DAY_NUMBER_FOR_UNIX_EPOCH}
    */
   public static final long CORRECT_CORRUPT_DATE_SHIFT = 2 * JULIAN_DAY_NUMBER_FOR_UNIX_EPOCH;
-  // The year 5000 (or 1106685 day from Unix epoch) is chosen as the threshold for auto-detecting
date corruption.
-  // This balances two possible cases of bad auto-correction. External tools writing dates
in the future will not
-  // be shifted unless they are past this threshold (and we cannot identify them as external
files based on the metadata).
-  // On the other hand, historical dates written with Drill wouldn't risk being incorrectly
shifted unless they were
-  // something like 10,000 years in the past.
   private static final Chronology UTC = org.joda.time.chrono.ISOChronology.getInstanceUTC();
+  /**
+   * The year 5000 (or 1106685 day from Unix epoch) is chosen as the threshold for auto-detecting
date corruption.
+   * This balances two possible cases of bad auto-correction. External tools writing dates
in the future will not
+   * be shifted unless they are past this threshold (and we cannot identify them as external
files based on the metadata).
+   * On the other hand, historical dates written with Drill wouldn't risk being incorrectly
shifted unless they were
+   * something like 10,000 years in the past.
+   */
   public static final int DATE_CORRUPTION_THRESHOLD =
       (int) (UTC.getDateTimeMillis(5000, 1, 1, 0) / DateTimeConstants.MILLIS_PER_DAY);
-
+  /**
+   * Version 2 (and later) of the Drill Parquet writer uses the date format described in
the
+   * <a href="https://github.com/Parquet/parquet-format/blob/master/LogicalTypes.md#date">Parquet
spec</a>.
+   * Prior versions had dates formatted with {@link org.apache.drill.exec.store.parquet.ParquetReaderUtility#CORRECT_CORRUPT_DATE_SHIFT}
+   */
+  public static final int DRILL_WRITER_VERSION_STD_DATE_FORMAT = 2;
   /**
    * For most recently created parquet files, we can determine if we have corrupted dates
(see DRILL-4203)
    * based on the file metadata. For older files that lack statistics we must actually test
the values
@@ -130,10 +137,9 @@ public class ParquetReaderUtility {
   }
 
   public static void correctDatesInMetadataCache(Metadata.ParquetTableMetadataBase parquetTableMetadata)
{
-    boolean isDateCorrect = parquetTableMetadata.isDateCorrect();
-    DateCorruptionStatus cacheFileContainsCorruptDates = isDateCorrect ?
-        DateCorruptionStatus.META_SHOWS_NO_CORRUPTION : DateCorruptionStatus.META_SHOWS_CORRUPTION;
-    if (cacheFileContainsCorruptDates == DateCorruptionStatus.META_SHOWS_CORRUPTION) {
+    DateCorruptionStatus cacheFileCanContainsCorruptDates = parquetTableMetadata instanceof
Metadata.ParquetTableMetadata_v3 ?
+        DateCorruptionStatus.META_SHOWS_NO_CORRUPTION : DateCorruptionStatus.META_UNCLEAR_TEST_VALUES;
+    if (cacheFileCanContainsCorruptDates == DateCorruptionStatus.META_UNCLEAR_TEST_VALUES)
{
       // Looking for the DATE data type of column names in the metadata cache file ("metadata_version"
: "v2")
       String[] names = new String[0];
       if (parquetTableMetadata instanceof Metadata.ParquetTableMetadata_v2) {
@@ -189,9 +195,20 @@ public class ParquetReaderUtility {
 
     String createdBy = footer.getFileMetaData().getCreatedBy();
     String drillVersion = footer.getFileMetaData().getKeyValueMetaData().get(ParquetRecordWriter.DRILL_VERSION_PROPERTY);
-    String isDateCorrect = footer.getFileMetaData().getKeyValueMetaData().get(ParquetRecordWriter.IS_DATE_CORRECT_PROPERTY);
+    String stringWriterVersion = footer.getFileMetaData().getKeyValueMetaData().get(ParquetRecordWriter.WRITER_VERSION_PROPERTY);
+    // This flag can be present in parquet files which were generated with 1.9.0-SNAPSHOT
and 1.9.0 drill versions.
+    // If this flag is present it means that the version of the drill parquet writer is 2
+    final String isDateCorrectFlag = "is.date.correct";
+    String isDateCorrect = footer.getFileMetaData().getKeyValueMetaData().get(isDateCorrectFlag);
     if (drillVersion != null) {
-      return Boolean.valueOf(isDateCorrect) ? DateCorruptionStatus.META_SHOWS_NO_CORRUPTION
+      int writerVersion = 1;
+      if (stringWriterVersion != null) {
+        writerVersion = Integer.parseInt(stringWriterVersion);
+      }
+      else if (Boolean.valueOf(isDateCorrect)) {
+        writerVersion = DRILL_WRITER_VERSION_STD_DATE_FORMAT;
+      }
+      return writerVersion >= DRILL_WRITER_VERSION_STD_DATE_FORMAT ? DateCorruptionStatus.META_SHOWS_NO_CORRUPTION
           : DateCorruptionStatus.META_SHOWS_CORRUPTION;
     } else {
       // Possibly an old, un-migrated Drill file, check the column statistics to see if min/max
values look corrupt
@@ -261,7 +278,8 @@ public class ParquetReaderUtility {
         // this reader only supports flat data, this is restricted in the ParquetScanBatchCreator
         // creating a NameSegment makes sure we are using the standard code for comparing
names,
         // currently it is all case-insensitive
-        if (AbstractRecordReader.isStarQuery(columns) || new PathSegment.NameSegment(column.getPath()[0]).equals(schemaPath.getRootSegment()))
{
+        if (AbstractRecordReader.isStarQuery(columns)
+            || new PathSegment.NameSegment(column.getPath()[0]).equals(schemaPath.getRootSegment()))
{
           int colIndex = -1;
           ConvertedType convertedType = schemaElements.get(column.getPath()[0]).getConverted_type();
           if (convertedType != null && convertedType.equals(ConvertedType.DATE))
{

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index bc6bb65..4ee863a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -78,7 +78,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
   private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
 
   public static final String DRILL_VERSION_PROPERTY = "drill.version";
-  public static final String IS_DATE_CORRECT_PROPERTY = "is.date.correct";
+  public static final String WRITER_VERSION_PROPERTY = "drill-writer.version";
 
   private ParquetFileWriter parquetFileWriter;
   private MessageType schema;
@@ -116,7 +116,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
     this.partitionColumns = writer.getPartitionColumns();
     this.hasPartitions = partitionColumns != null && partitionColumns.size() >
0;
     this.extraMetaData.put(DRILL_VERSION_PROPERTY, DrillVersionInfo.getVersion());
-    this.extraMetaData.put(IS_DATE_CORRECT_PROPERTY, "true");
+    this.extraMetaData.put(WRITER_VERSION_PROPERTY, String.valueOf(ParquetWriter.WRITER_VERSION));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
index 49c231e..716c56d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
@@ -40,6 +40,18 @@ import com.google.common.base.Preconditions;
 public class ParquetWriter extends AbstractWriter {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetWriter.class);
 
+/** Version of Drill's Parquet writer. Increment this version (by 1) any time we make any
format change to the file.
+ * Format changes include:
+ * <ul>
+ * <li>Supporting new data types,
+ * <li>Changes to the format of data fields,
+ * <li>Adding new metadata to the file footer, etc.
+ * </ul>
+ * Newer readers must be able to read old files. The Writer version tells the Parquet reader
how to interpret fields
+ * or metadata when that data changes format from one writer version to another.
+ */
+  public static final int WRITER_VERSION = 2;
+
   private final String location;
   private final List<String> partitionColumns;
   private final ParquetFormatPlugin formatPlugin;

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
index 27cc093..0ab247d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
@@ -67,8 +67,8 @@ import java.util.regex.Pattern;
 public class TestCorruptParquetDateCorrection extends PlanTestBase {
 
   // 4 files are in the directory:
-  //    - one created with the fixed version of the reader
-  //        - files have extra meta field: is.date.correct = true
+  //    - one created with the parquet-writer version number of "2"
+  //        - files have extra meta field: parquet-writer.version = 2
   //    - one from and old version of Drill, before we put in proper created by in metadata
   //        - this is read properly by looking at a Max value in the file statistics, to
see that
   //          it is way off of a typical date value
@@ -123,7 +123,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
 
   /**
    * Test reading a directory full of partitioned parquet files with dates, these files have
a drill version
-   * number of 1.9.0-SNAPSHOT and is.date.correct = true label in their footers, so we can
be certain
+   * number of "1.9.0-SNAPSHOT" and parquet-writer version number of "2" in their footers,
so we can be certain
    * they do not have corruption. The option to disable the correction is passed, but it
will not change the result
    * in the case where we are certain correction is NOT needed. For more info see DRILL-4203.
    */

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet
index 0ec76c8..34224c1 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet
index b43ab9d..8ce72e1 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet
index 95d46bb..b96fa61 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet
index 9f7269e..3c5a13a 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet
index b0d370a..3a6db56 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet
index a5c7b86..6e053ee 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet
index 3d46f56..7a461d0 100644
Binary files a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet
and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/cf29b917/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt
b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt
index bb6e282..54f18a8 100644
--- a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt
+++ b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt
@@ -297,5 +297,5 @@
     } ]
   } ],
   "directories" : [ "file:REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption",
"file:REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203" ],
-  "isDateCorrect" : "true"
+  "drillVersion" : "1.9.0-SNAPSHOT"
 }


Mime
View raw message