drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [2/4] drill git commit: DRILL-4103: add drill.version to parquet metadata
Date Thu, 26 Nov 2015 04:03:58 GMT
DRILL-4103: add drill.version to parquet metadata

This closes #264


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/56d5f794
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/56d5f794
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/56d5f794

Branch: refs/heads/master
Commit: 56d5f79453475bb144d3b7118ec6751c34d26f80
Parents: f7786cc
Author: Julien Le Dem <julien@ledem.net>
Authored: Tue Nov 17 17:19:35 2015 -0800
Committer: Jason Altekruse <altekrusejason@gmail.com>
Committed: Wed Nov 25 15:20:35 2015 -0800

----------------------------------------------------------------------
 .../exec/store/parquet/ParquetRecordWriter.java  |  4 ++++
 .../physical/impl/writer/TestParquetWriter.java  | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/56d5f794/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index a2cd1fa..710edd6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.util.DrillVersionInfo;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.exception.OutOfMemoryException;
 import org.apache.drill.exec.ops.FragmentContext;
@@ -76,6 +77,8 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
   private static final int MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
   private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
 
+  public static final String DRILL_VERSION_PROPERTY = "drill.version";
+
   private ParquetFileWriter parquetFileWriter;
   private MessageType schema;
   private Map<String, String> extraMetaData = new HashMap<>();
@@ -111,6 +114,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
         new ParquetDirectByteBufferAllocator(oContext.getAllocator()), pageSize);
     this.partitionColumns = writer.getPartitionColumns();
     this.hasPartitions = partitionColumns != null && partitionColumns.size() >
0;
+    this.extraMetaData.put(DRILL_VERSION_PROPERTY, DrillVersionInfo.getVersion());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/56d5f794/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 007561d..6890394 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -17,6 +17,10 @@
  */
 package org.apache.drill.exec.physical.impl.writer;
 
+import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
+import static org.junit.Assert.assertEquals;
+
 import java.io.File;
 import java.io.FileWriter;
 import java.math.BigDecimal;
@@ -28,12 +32,19 @@ import java.util.Map;
 
 import com.google.common.base.Joiner;
 import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.DrillVersionInfo;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.fn.interp.TestConstantFolding;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.store.parquet.ParquetRecordWriter;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.joda.time.DateTime;
 import org.joda.time.Period;
 import org.junit.AfterClass;
@@ -714,6 +725,14 @@ public class TestParquetWriter extends BaseTestQuery {
           .sqlBaselineQuery(validateQuery)
           .go();
 
+      Configuration hadoopConf = new Configuration();
+      Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile);
+      FileSystem fs = output.getFileSystem(hadoopConf);
+      for (FileStatus file : fs.listStatus(output)) {
+        ParquetMetadata footer = ParquetFileReader.readFooter(hadoopConf, file, SKIP_ROW_GROUPS);
+        String version = footer.getFileMetaData().getKeyValueMetaData().get(DRILL_VERSION_PROPERTY);
+        assertEquals(DrillVersionInfo.getVersion(), version);
+      }
     } finally {
       deleteTableIfExists(outputFile);
     }


Mime
View raw message