drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject drill git commit: DRILL-2253: Vectorized Parquet reader fails to read correctly against RLE Dictionary encoded DATE column
Date Wed, 25 Feb 2015 00:37:27 GMT
Repository: drill
Updated Branches:
  refs/heads/master c8d2fe1b8 -> 4c7519aa0


DRILL-2253: Vectorized Parquet reader fails to read correctly against RLE Dictionary encoded
DATE column


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4c7519aa
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4c7519aa
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4c7519aa

Branch: refs/heads/master
Commit: 4c7519aa0a38fde50119266a9e3089a42b9054d0
Parents: c8d2fe1
Author: adeneche <adeneche@gmail.com>
Authored: Tue Feb 17 16:58:11 2015 -0800
Committer: Jason Altekruse <altekrusejason@gmail.com>
Committed: Tue Feb 24 16:15:08 2015 -0800

----------------------------------------------------------------------
 .../NullableFixedByteAlignedReaders.java        |   9 ++++-
 .../TestNullableFixedAlignedReaders.java        |  36 +++++++++++++++++++
 .../resources/parquet/date_dictionary.parquet   | Bin 0 -> 2008 bytes
 3 files changed, 44 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index 63ac0ee..707bc9c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -188,7 +188,14 @@ public class NullableFixedByteAlignedReaders {
 
     @Override
     void addNext(int start, int index) {
-      dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(readIntLittleEndian(bytebuf,
start) - ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5));
+      int intValue;
+      if (usingDictionary) {
+        intValue =  pageReader.dictionaryValueReader.readInteger();
+      } else {
+        intValue = readIntLittleEndian(bytebuf, start);
+      }
+
+      dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(intValue - ParquetOutputRecordWriter.JULIAN_DAY_EPOC
- 0.5));
     }
 
     // copied out of parquet library, didn't want to deal with the uneeded throws statement
they had declared

http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java
new file mode 100644
index 0000000..7e83482
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet.columnreaders;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Test;
+
+public class TestNullableFixedAlignedReaders extends BaseTestQuery {
+
+  /**
+   * check if NullableDateReader works well with dictionary encoding.
+   */
+  @Test
+  public void testNullableDateReaderWithDictionary() throws Exception {
+    // the file 'date_dictionary.parquet' contains one single DATE column with 600 rows and
290 distinct date values
+    // and uses the PLAIN_DICTIONARY encoder
+
+    // query parquet file. We shouldn't get any exception
+    testNoResult("SELECT * FROM cp.`parquet/date_dictionary.parquet`");
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet
new file mode 100644
index 0000000..e1adfc0
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet
differ


Mime
View raw message