Return-Path: X-Original-To: apmail-drill-commits-archive@www.apache.org Delivered-To: apmail-drill-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 92BDC18232 for ; Tue, 9 Feb 2016 00:26:52 +0000 (UTC) Received: (qmail 21726 invoked by uid 500); 9 Feb 2016 00:26:52 -0000 Delivered-To: apmail-drill-commits-archive@drill.apache.org Received: (qmail 21654 invoked by uid 500); 9 Feb 2016 00:26:52 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 21562 invoked by uid 99); 9 Feb 2016 00:26:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 09 Feb 2016 00:26:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3E6BAE56E3; Tue, 9 Feb 2016 00:26:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: json@apache.org To: commits@drill.apache.org Date: Tue, 09 Feb 2016 00:26:55 -0000 Message-Id: <6e885abeb1b24a99b558ee606ed60cff@git.apache.org> In-Reply-To: <0e099b992bb64183a0ab11c9e8b5bcde@git.apache.org> References: <0e099b992bb64183a0ab11c9e8b5bcde@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [4/8] drill git commit: DRILL-4314: Unit Test Framework can support schema check DRILL-4314: Unit Test Framework can support schema check Closes #339 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bd14e1ad Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bd14e1ad Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bd14e1ad Branch: refs/heads/master Commit: bd14e1ad83034cca447cf29c7ed3125334f03ee3 Parents: 4e9b825 Author: Hsuan-Yi Chu Authored: Tue Jan 26 16:33:39 2016 -0800 Committer: Jason Altekruse Committed: Mon Feb 8 14:55:00 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/drill/DrillTestWrapper.java | 51 +++++++- .../test/java/org/apache/drill/TestBuilder.java | 57 +++++++++ .../org/apache/drill/TestFrameworkTest.java | 61 ++++++++++ .../drill/TestFunctionsWithTypeExpoQueries.java | 118 +++++++++++++++++++ 4 files changed, 283 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java index df2cfb0..35e668c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java @@ -119,10 +119,14 @@ public class DrillTestWrapper { } public void run() throws Exception { - if (ordered) { - compareOrderedResults(); + if (testBuilder.getExpectedSchema() != null) { + compareSchemaOnly(); } else { - compareUnorderedResults(); + if (ordered) { + compareOrderedResults(); + } else { + compareUnorderedResults(); + } } } @@ -295,6 +299,45 @@ public class DrillTestWrapper { return combinedVectors; } + protected void compareSchemaOnly() throws Exception { + RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + List actual = Collections.EMPTY_LIST; + + + QueryDataBatch batch = null; + try { + BaseTestQuery.test(testOptionSettingQueries); + actual = BaseTestQuery.testRunAndReturn(queryType, query); + batch = actual.get(0); + loader.load(batch.getHeader().getDef(), batch.getData()); + + final BatchSchema schema = loader.getSchema(); + if(schema.getFieldCount() != testBuilder.getExpectedSchema().size()) { + throw new Exception("The column numbers for actual schema and expected schema do not match"); + } + + for(int i = 0; i < schema.getFieldCount(); ++i) { + final SchemaPath actualSchemaPath = schema.getColumn(i).getPath(); + final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType(); + + final SchemaPath expectedSchemaPath = schema.getColumn(i).getPath(); + final TypeProtos.MajorType expectedlMajorType = schema.getColumn(i).getType(); + + if(!actualSchemaPath.equals(expectedSchemaPath) + || !actualMajorType.equals(expectedlMajorType)) { + throw new Exception("The type of the " + i + "-th column is '" + actualSchemaPath + "' mismatched, expected: '" + + expectedlMajorType + "'"); + } + } + + } finally { + if(batch != null) { + batch.release(); + } + loader.clear(); + } + } + /** * Use this method only if necessary to validate one query against another. If you are just validating against a * baseline file use one of the simpler interfaces that will write the validation query for you. @@ -305,7 +348,7 @@ public class DrillTestWrapper { RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); BatchSchema schema = null; - List actual = Collections.EMPTY_LIST;; + List actual = Collections.EMPTY_LIST; List expected = Collections.EMPTY_LIST; List expectedRecords = new ArrayList<>(); List actualRecords = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java index 9115a6c..f4a5825 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java @@ -28,6 +28,7 @@ import java.util.Map; import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CommonTokenStream; import org.antlr.runtime.RecognitionException; +import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.parser.ExprLexer; import org.apache.drill.common.expression.parser.ExprParser; @@ -125,6 +126,10 @@ public class TestBuilder { getValidationQueryType(), ordered, approximateEquality, highPerformanceComparison, baselineRecords, expectedNumBatches); } + public List> getExpectedSchema() { + return null; + } + public void go() throws Exception { build().run(); } @@ -235,6 +240,20 @@ public class TestBuilder { expectedNumBatches); } + public SchemaTestBuilder schemaBaseLine(List> expectedSchema) { + assert expectedSchema != null : "The expected schema can be provided once"; + assert baselineColumns == null : "The column information should be captured in expected schema, not baselineColumns"; + assert baselineValues == null && baselineRecords == null : "Since only schema will be compared in this test, no record is expected"; + + return new SchemaTestBuilder( + allocator, + query, + queryType, + baselineOptionSettingQueries, + testOptionSettingQueries, + expectedSchema); + } + public TestBuilder baselineTypes(Map baselineTypeMap) { this.baselineTypeMap = baselineTypeMap; return this; @@ -277,6 +296,7 @@ public class TestBuilder { * @return */ public TestBuilder baselineValues(Object ... baselineValues) { + assert getExpectedSchema() == null : "The expected schema is not needed when baselineValues are provided "; if (ordered == null) { throw new RuntimeException("Ordering not set, before specifying baseline data you must explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName()); } @@ -329,6 +349,7 @@ public class TestBuilder { * be used to create a map for the one record verification. */ public TestBuilder baselineColumns(String... columns) { + assert getExpectedSchema() == null : "The expected schema is not needed when baselineColumns are provided "; for (int i = 0; i < columns.length; i++) { columns[i] = parsePath(columns[i]).toExpr(); } @@ -469,7 +490,43 @@ public class TestBuilder { protected UserBitShared.QueryType getValidationQueryType() throws Exception { return UserBitShared.QueryType.SQL; } + } + + public class SchemaTestBuilder extends TestBuilder { + private List> expectedSchema; + SchemaTestBuilder(BufferAllocator allocator, String query, UserBitShared.QueryType queryType, + String baselineOptionSettingQueries, String testOptionSettingQueries, List> expectedSchema) { + super(allocator, query, queryType, false, false, null, baselineOptionSettingQueries, testOptionSettingQueries, false, -1); + expectsEmptyResultSet(); + this.expectedSchema = expectedSchema; + } + + public TestBuilder baselineColumns(String... columns) { + assert false : "The column information should be captured in expected scheme, not baselineColumns"; + return this; + } + @Override + public TestBuilder baselineRecords(List materializedRecords) { + assert false : "Since only schema will be compared in this test, no record is expected"; + return this; + } + + @Override + public TestBuilder baselineValues(Object[] objects) { + assert false : "Since only schema will be compared in this test, no record is expected"; + return this; + } + + @Override + protected UserBitShared.QueryType getValidationQueryType() throws Exception { + return null; + } + + @Override + public List> getExpectedSchema() { + return expectedSchema; + } } public class JSONTestBuilder extends TestBuilder { http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java index f6c8859..3771edd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java @@ -24,9 +24,13 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; @@ -42,6 +46,63 @@ public class TestFrameworkTest extends BaseTestQuery{ private static String CSV_COLS = " cast(columns[0] as bigint) employee_id, columns[1] as first_name, columns[2] as last_name "; + @Test(expected = AssertionError.class) + public void testSchemaTestBuilderSetInvalidBaselineValues() throws Exception { + final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0"; + + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.VARCHAR) + .setMode(TypeProtos.DataMode.REQUIRED) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .schemaBaseLine(expectedSchema) + .baselineValues(new Object[0]) + .build() + .run(); + } + + @Test(expected = AssertionError.class) + public void testSchemaTestBuilderSetInvalidBaselineRecords() throws Exception { + final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0"; + + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.VARCHAR) + .setMode(TypeProtos.DataMode.REQUIRED) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .schemaBaseLine(expectedSchema) + .baselineRecords(new ArrayList()) + .build() + .run(); + } + + @Test(expected = AssertionError.class) + public void testSchemaTestBuilderSetInvalidBaselineColumns() throws Exception { + final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0"; + + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.VARCHAR) + .setMode(TypeProtos.DataMode.REQUIRED) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .baselineColumns("col") + .schemaBaseLine(expectedSchema) + .build() + .run(); + } + @Test public void testCSVVerification() throws Exception { testBuilder() http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java new file mode 100644 index 0000000..7d3f6d0 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.util.FileUtils; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery { + @Test + public void testConcatWithMoreThanTwoArgs() throws Exception { + final String query = "select concat(r_name, r_name, r_name) as col \n" + + "from cp.`tpch/region.parquet` limit 0"; + + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.VARCHAR) + .setMode(TypeProtos.DataMode.REQUIRED) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .schemaBaseLine(expectedSchema) + .build() + .run(); + } + + @Test + public void testTrimOnlyOneArg() throws Exception { + final String query1 = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0"; + final String query2 = "SELECT rtrim('drill') as col FROM (VALUES(1)) limit 0"; + final String query3 = "SELECT btrim('drill') as col FROM (VALUES(1)) limit 0"; + + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.VARCHAR) + .setMode(TypeProtos.DataMode.REQUIRED) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query1) + .schemaBaseLine(expectedSchema) + .build() + .run(); + + testBuilder() + .sqlQuery(query2) + .schemaBaseLine(expectedSchema) + .build() + .run(); + + testBuilder() + .sqlQuery(query3) + .schemaBaseLine(expectedSchema) + .build() + .run(); + } + + @Test + public void testExtract() throws Exception { + final String query = "select extract(second from time '02:30:45.100') as col \n" + + "from cp.`employee.json` limit 0"; + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.FLOAT8) + .setMode(TypeProtos.DataMode.OPTIONAL) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .schemaBaseLine(expectedSchema) + .build() + .run(); + } + + @Test + public void tesIsNull() throws Exception { + final String query = "select r_name is null as col from cp.`tpch/region.parquet` limit 0"; + List> expectedSchema = Lists.newArrayList(); + TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder() + .setMinorType(TypeProtos.MinorType.BIT) + .setMode(TypeProtos.DataMode.OPTIONAL) + .build(); + expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType)); + + testBuilder() + .sqlQuery(query) + .schemaBaseLine(expectedSchema) + .build() + .run(); + } +}