Return-Path: X-Original-To: apmail-drill-commits-archive@www.apache.org Delivered-To: apmail-drill-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6335917EAD for ; Wed, 25 Feb 2015 00:37:27 +0000 (UTC) Received: (qmail 72226 invoked by uid 500); 25 Feb 2015 00:37:27 -0000 Delivered-To: apmail-drill-commits-archive@drill.apache.org Received: (qmail 72189 invoked by uid 500); 25 Feb 2015 00:37:27 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 72176 invoked by uid 99); 25 Feb 2015 00:37:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Feb 2015 00:37:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1AB33E0388; Wed, 25 Feb 2015 00:37:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: json@apache.org To: commits@drill.apache.org Message-Id: <5d1c996e2ef24b5aa2b89721911aa862@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: drill git commit: DRILL-2253: Vectorized Parquet reader fails to read correctly against RLE Dictionary encoded DATE column Date: Wed, 25 Feb 2015 00:37:27 +0000 (UTC) Repository: drill Updated Branches: refs/heads/master c8d2fe1b8 -> 4c7519aa0 DRILL-2253: Vectorized Parquet reader fails to read correctly against RLE Dictionary encoded DATE column Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4c7519aa Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4c7519aa Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4c7519aa Branch: refs/heads/master Commit: 4c7519aa0a38fde50119266a9e3089a42b9054d0 Parents: c8d2fe1 Author: adeneche Authored: Tue Feb 17 16:58:11 2015 -0800 Committer: Jason Altekruse Committed: Tue Feb 24 16:15:08 2015 -0800 ---------------------------------------------------------------------- .../NullableFixedByteAlignedReaders.java | 9 ++++- .../TestNullableFixedAlignedReaders.java | 36 +++++++++++++++++++ .../resources/parquet/date_dictionary.parquet | Bin 0 -> 2008 bytes 3 files changed, 44 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java index 63ac0ee..707bc9c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java @@ -188,7 +188,14 @@ public class NullableFixedByteAlignedReaders { @Override void addNext(int start, int index) { - dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(readIntLittleEndian(bytebuf, start) - ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5)); + int intValue; + if (usingDictionary) { + intValue = pageReader.dictionaryValueReader.readInteger(); + } else { + intValue = readIntLittleEndian(bytebuf, start); + } + + dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(intValue - ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5)); } // copied out of parquet library, didn't want to deal with the uneeded throws statement they had declared http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java new file mode 100644 index 0000000..7e83482 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestNullableFixedAlignedReaders.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet.columnreaders; + +import org.apache.drill.BaseTestQuery; +import org.junit.Test; + +public class TestNullableFixedAlignedReaders extends BaseTestQuery { + + /** + * check if NullableDateReader works well with dictionary encoding. + */ + @Test + public void testNullableDateReaderWithDictionary() throws Exception { + // the file 'date_dictionary.parquet' contains one single DATE column with 600 rows and 290 distinct date values + // and uses the PLAIN_DICTIONARY encoder + + // query parquet file. We shouldn't get any exception + testNoResult("SELECT * FROM cp.`parquet/date_dictionary.parquet`"); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/4c7519aa/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet new file mode 100644 index 0000000..e1adfc0 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/date_dictionary.parquet differ