Return-Path: X-Original-To: apmail-drill-commits-archive@www.apache.org Delivered-To: apmail-drill-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DA5A6175EA for ; Mon, 30 Mar 2015 18:27:28 +0000 (UTC) Received: (qmail 80441 invoked by uid 500); 30 Mar 2015 18:26:30 -0000 Delivered-To: apmail-drill-commits-archive@drill.apache.org Received: (qmail 80407 invoked by uid 500); 30 Mar 2015 18:26:30 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 80398 invoked by uid 99); 30 Mar 2015 18:26:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Mar 2015 18:26:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 31238E03C7; Mon, 30 Mar 2015 18:26:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mehant@apache.org To: commits@drill.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: drill git commit: DRILL-2375: implement reader reset mechanism and reset reader before accessing it during projection Date: Mon, 30 Mar 2015 18:26:30 +0000 (UTC) Repository: drill Updated Branches: refs/heads/master 96d51bded -> 0fbcddba1 DRILL-2375: implement reader reset mechanism and reset reader before accessing it during projection Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0fbcddba Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0fbcddba Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0fbcddba Branch: refs/heads/master Commit: 0fbcddba14405ec94d51b0ba3512925168efb433 Parents: 96d51bd Author: Hanifi Gunes Authored: Mon Mar 23 19:38:23 2015 -0700 Committer: Mehant Baid Committed: Mon Mar 30 10:27:02 2015 -0700 ---------------------------------------------------------------------- .../src/main/codegen/templates/BaseReader.java | 1 + .../drill/exec/expr/EvaluationVisitor.java | 1 + .../vector/complex/impl/AbstractBaseReader.java | 5 +++++ .../complex/impl/RepeatedListReaderImpl.java | 11 +++++++++++ .../complex/impl/RepeatedMapReaderImpl.java | 11 +++++++++++ .../expr/TestSchemaPathMaterialization.java | 20 +++++++++++++++++--- 6 files changed, 46 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/main/codegen/templates/BaseReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/BaseReader.java b/exec/java-exec/src/main/codegen/templates/BaseReader.java index 9281d3c..116deda 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseReader.java +++ b/exec/java-exec/src/main/codegen/templates/BaseReader.java @@ -32,6 +32,7 @@ package org.apache.drill.exec.vector.complex.reader; public interface BaseReader extends Positionable{ MajorType getType(); MaterializedField getField(); + void reset(); public interface MapReader extends BaseReader, Iterable{ FieldReader reader(String name); http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index a5758fb..c20ff14 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -373,6 +373,7 @@ public class EvaluationVisitor { JBlock eval = generator.getEvalBlock().block(); // position to the correct value. + eval.add(expr.invoke("reset")); eval.add(expr.invoke("setPosition").arg(indexVariable)); int listNum = 0; http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java index 04235fa..22addc9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java @@ -44,6 +44,11 @@ abstract class AbstractBaseReader implements FieldReader{ } @Override + public void reset() { + index = 0; + } + + @Override public Iterator iterator() { throw new IllegalStateException("The current reader doesn't support reading as a map."); } http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java index c51dfda..805ba89 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java @@ -68,6 +68,17 @@ public class RepeatedListReaderImpl extends AbstractFieldReader{ private int maxOffset; @Override + public void reset() { + super.reset(); + currentOffset = 0; + maxOffset = 0; + if (reader != null) { + reader.reset(); + } + reader = null; + } + + @Override public int size() { return maxOffset - currentOffset; } http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java index 9136277..1cee547 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java @@ -76,6 +76,17 @@ public class RepeatedMapReaderImpl extends AbstractFieldReader{ private int maxOffset; @Override + public void reset() { + super.reset(); + currentOffset = 0; + maxOffset = 0; + for (FieldReader reader:fields.values()) { + reader.reset(); + } + fields.clear(); + } + + @Override public int size() { if (isNull()) { return 0; http://git-wip-us.apache.org/repos/asf/drill/blob/0fbcddba/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java index fb4cad9..66947f1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java @@ -38,9 +38,23 @@ public class TestSchemaPathMaterialization extends BaseTestQuery { } @Test - @Ignore("Ignored until DRILL-2375 is fixed") + public void testMultiProjectionFromMultiLevelRepeatedListWhenFieldsExist() throws Exception { + final String query = "select t.odd[0][0][0] v1, t.odd[0][1][0] v2, t.odd[0][2][0] v3 " + + " from cp.`complex/json/repeated_list.json` t"; + + testRunAndPrint(UserBitShared.QueryType.SQL, query); + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("v1", "v2", "v3") + .baselineValues(1L, null, 3L) + .go(); + } + + @Test + @Ignore("Ignored until DRILL-2539 is fixed") public void testProjectionFromMultiLevelRepeatedList() throws Exception { - final String query = "select t.odd[0][0][0] v1, t.odd[0][0][1] v2, t.odd[0][0][2] v3, " + + final String query = "select t.odd[0][1][0] v1, t.odd[0][1][0] v2, t.odd[0][2][0] v3, " + " t.odd[1] v4, t.odd[2][0][0] v5, t.odd[2][1][0] v6" + " from cp.`complex/json/repeated_list.json` t"; @@ -54,7 +68,7 @@ public class TestSchemaPathMaterialization extends BaseTestQuery { } @Test - @Ignore("Ignored until DRILL-2375 is fixed") + @Ignore("Ignored until DRILL-2539 is fixed") public void testProjectionFromMultiLevelRepeatedListMap() throws Exception { final String query = "select t.odd[0][0].val[0] v1, t.odd[0][0].val[0] v2, " + " from cp.`complex/json/repeated_list_map.json` t";