drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [1/3] drill git commit: DRILL-4103: add drill.version to parquet metadata
Date Wed, 18 Nov 2015 05:59:56 GMT
Repository: drill
Updated Branches:
  refs/heads/1.3.0 e465cd90c -> 45d0326cc


DRILL-4103: add drill.version to parquet metadata


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a2896681
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a2896681
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a2896681

Branch: refs/heads/1.3.0
Commit: a2896681769ac64b5935db6b09c5b0978f05d2f1
Parents: e465cd9
Author: Julien Le Dem <julien@ledem.net>
Authored: Tue Nov 17 17:19:35 2015 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Tue Nov 17 19:31:26 2015 -0800

----------------------------------------------------------------------
 .../exec/store/parquet/ParquetRecordWriter.java  |  4 ++++
 .../physical/impl/writer/TestParquetWriter.java  | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a2896681/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index f784566..0f0cd1c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.util.DrillVersionInfo;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.apache.drill.exec.ops.FragmentContext;
@@ -77,6 +78,8 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
   private static final int MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
   private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
 
+  public static final String DRILL_VERSION_PROPERTY = "drill.version";
+
   private ParquetFileWriter parquetFileWriter;
   private MessageType schema;
   private Map<String, String> extraMetaData = new HashMap<>();
@@ -112,6 +115,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
         new ParquetDirectByteBufferAllocator(oContext.getAllocator()), pageSize);
     this.partitionColumns = writer.getPartitionColumns();
     this.hasPartitions = partitionColumns != null && partitionColumns.size() >
0;
+    this.extraMetaData.put(DRILL_VERSION_PROPERTY, DrillVersionInfo.getVersion());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/a2896681/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 007561d..6890394 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -17,6 +17,10 @@
  */
 package org.apache.drill.exec.physical.impl.writer;
 
+import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
+import static org.junit.Assert.assertEquals;
+
 import java.io.File;
 import java.io.FileWriter;
 import java.math.BigDecimal;
@@ -28,12 +32,19 @@ import java.util.Map;
 
 import com.google.common.base.Joiner;
 import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.DrillVersionInfo;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.fn.interp.TestConstantFolding;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.store.parquet.ParquetRecordWriter;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.joda.time.DateTime;
 import org.joda.time.Period;
 import org.junit.AfterClass;
@@ -714,6 +725,14 @@ public class TestParquetWriter extends BaseTestQuery {
           .sqlBaselineQuery(validateQuery)
           .go();
 
+      Configuration hadoopConf = new Configuration();
+      Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile);
+      FileSystem fs = output.getFileSystem(hadoopConf);
+      for (FileStatus file : fs.listStatus(output)) {
+        ParquetMetadata footer = ParquetFileReader.readFooter(hadoopConf, file, SKIP_ROW_GROUPS);
+        String version = footer.getFileMetaData().getKeyValueMetaData().get(DRILL_VERSION_PROPERTY);
+        assertEquals(DrillVersionInfo.getVersion(), version);
+      }
     } finally {
       deleteTableIfExists(outputFile);
     }


Mime
View raw message