drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [4/8] drill git commit: DRILL-4314: Unit Test Framework can support schema check
Date Tue, 09 Feb 2016 00:26:55 GMT
DRILL-4314: Unit Test Framework can support schema check

Closes #339


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bd14e1ad
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bd14e1ad
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bd14e1ad

Branch: refs/heads/master
Commit: bd14e1ad83034cca447cf29c7ed3125334f03ee3
Parents: 4e9b825
Author: Hsuan-Yi Chu <hsuanyi@usc.edu>
Authored: Tue Jan 26 16:33:39 2016 -0800
Committer: Jason Altekruse <altekrusejason@gmail.com>
Committed: Mon Feb 8 14:55:00 2016 -0800

----------------------------------------------------------------------
 .../java/org/apache/drill/DrillTestWrapper.java |  51 +++++++-
 .../test/java/org/apache/drill/TestBuilder.java |  57 +++++++++
 .../org/apache/drill/TestFrameworkTest.java     |  61 ++++++++++
 .../drill/TestFunctionsWithTypeExpoQueries.java | 118 +++++++++++++++++++
 4 files changed, 283 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
index df2cfb0..35e668c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
@@ -119,10 +119,14 @@ public class DrillTestWrapper {
   }
 
   public void run() throws Exception {
-    if (ordered) {
-      compareOrderedResults();
+    if (testBuilder.getExpectedSchema() != null) {
+      compareSchemaOnly();
     } else {
-      compareUnorderedResults();
+      if (ordered) {
+        compareOrderedResults();
+      } else {
+        compareUnorderedResults();
+      }
     }
   }
 
@@ -295,6 +299,45 @@ public class DrillTestWrapper {
     return combinedVectors;
   }
 
+  protected void compareSchemaOnly() throws Exception {
+    RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+    List<QueryDataBatch> actual = Collections.EMPTY_LIST;
+
+
+    QueryDataBatch batch = null;
+    try {
+      BaseTestQuery.test(testOptionSettingQueries);
+      actual = BaseTestQuery.testRunAndReturn(queryType, query);
+      batch = actual.get(0);
+      loader.load(batch.getHeader().getDef(), batch.getData());
+
+      final BatchSchema schema = loader.getSchema();
+      if(schema.getFieldCount() != testBuilder.getExpectedSchema().size()) {
+        throw new Exception("The column numbers for actual schema and expected schema do
not match");
+      }
+
+      for(int i = 0; i < schema.getFieldCount(); ++i) {
+        final SchemaPath actualSchemaPath = schema.getColumn(i).getPath();
+        final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType();
+
+        final SchemaPath expectedSchemaPath = schema.getColumn(i).getPath();
+        final TypeProtos.MajorType expectedlMajorType = schema.getColumn(i).getType();
+
+        if(!actualSchemaPath.equals(expectedSchemaPath)
+            || !actualMajorType.equals(expectedlMajorType)) {
+          throw new Exception("The type of the " + i + "-th column is '" + actualSchemaPath
+ "' mismatched, expected: '"
+              + expectedlMajorType + "'");
+        }
+      }
+
+    }  finally {
+      if(batch != null) {
+        batch.release();
+      }
+      loader.clear();
+    }
+  }
+
   /**
    * Use this method only if necessary to validate one query against another. If you are
just validating against a
    * baseline file use one of the simpler interfaces that will write the validation query
for you.
@@ -305,7 +348,7 @@ public class DrillTestWrapper {
     RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
     BatchSchema schema = null;
 
-    List<QueryDataBatch> actual = Collections.EMPTY_LIST;;
+    List<QueryDataBatch> actual = Collections.EMPTY_LIST;
     List<QueryDataBatch> expected = Collections.EMPTY_LIST;
     List<Map> expectedRecords = new ArrayList<>();
     List<Map> actualRecords = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
index 9115a6c..f4a5825 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
@@ -28,6 +28,7 @@ import java.util.Map;
 import org.antlr.runtime.ANTLRStringStream;
 import org.antlr.runtime.CommonTokenStream;
 import org.antlr.runtime.RecognitionException;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.parser.ExprLexer;
 import org.apache.drill.common.expression.parser.ExprParser;
@@ -125,6 +126,10 @@ public class TestBuilder {
         getValidationQueryType(), ordered, approximateEquality, highPerformanceComparison,
baselineRecords, expectedNumBatches);
   }
 
+  public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
+    return null;
+  }
+
   public void go() throws Exception {
     build().run();
   }
@@ -235,6 +240,20 @@ public class TestBuilder {
         expectedNumBatches);
   }
 
+  public SchemaTestBuilder schemaBaseLine(List<Pair<SchemaPath, TypeProtos.MajorType>>
expectedSchema) {
+    assert expectedSchema != null : "The expected schema can be provided once";
+    assert baselineColumns == null : "The column information should be captured in expected
schema, not baselineColumns";
+    assert baselineValues == null && baselineRecords == null : "Since only schema
will be compared in this test, no record is expected";
+
+    return new SchemaTestBuilder(
+        allocator,
+        query,
+        queryType,
+        baselineOptionSettingQueries,
+        testOptionSettingQueries,
+        expectedSchema);
+  }
+
   public TestBuilder baselineTypes(Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap)
{
     this.baselineTypeMap = baselineTypeMap;
     return this;
@@ -277,6 +296,7 @@ public class TestBuilder {
    * @return
    */
   public TestBuilder baselineValues(Object ... baselineValues) {
+    assert getExpectedSchema() == null : "The expected schema is not needed when baselineValues
are provided ";
     if (ordered == null) {
       throw new RuntimeException("Ordering not set, before specifying baseline data you must
explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName());
     }
@@ -329,6 +349,7 @@ public class TestBuilder {
    * be used to create a map for the one record verification.
    */
   public TestBuilder baselineColumns(String... columns) {
+    assert getExpectedSchema() == null : "The expected schema is not needed when baselineColumns
are provided ";
     for (int i = 0; i < columns.length; i++) {
       columns[i] = parsePath(columns[i]).toExpr();
     }
@@ -469,7 +490,43 @@ public class TestBuilder {
     protected UserBitShared.QueryType getValidationQueryType() throws Exception {
       return UserBitShared.QueryType.SQL;
     }
+  }
+
+  public class SchemaTestBuilder extends TestBuilder {
+    private List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema;
+    SchemaTestBuilder(BufferAllocator allocator, String query, UserBitShared.QueryType queryType,
+        String baselineOptionSettingQueries, String testOptionSettingQueries, List<Pair<SchemaPath,
TypeProtos.MajorType>> expectedSchema) {
+      super(allocator, query, queryType, false, false, null, baselineOptionSettingQueries,
testOptionSettingQueries, false, -1);
+      expectsEmptyResultSet();
+      this.expectedSchema = expectedSchema;
+    }
+
+    public TestBuilder baselineColumns(String... columns) {
+      assert false : "The column information should be captured in expected scheme, not baselineColumns";
+      return this;
+    }
 
+    @Override
+    public TestBuilder baselineRecords(List<Map> materializedRecords) {
+      assert false : "Since only schema will be compared in this test, no record is expected";
+      return this;
+    }
+
+    @Override
+    public TestBuilder baselineValues(Object[] objects) {
+      assert false : "Since only schema will be compared in this test, no record is expected";
+      return this;
+    }
+
+    @Override
+    protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+      return null;
+    }
+
+    @Override
+    public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
+      return expectedSchema;
+    }
   }
 
   public class JSONTestBuilder extends TestBuilder {

http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
index f6c8859..3771edd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
@@ -24,9 +24,13 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import java.math.BigDecimal;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
@@ -42,6 +46,63 @@ public class TestFrameworkTest extends BaseTestQuery{
 
   private static String CSV_COLS = " cast(columns[0] as bigint) employee_id, columns[1] as
first_name, columns[2] as last_name ";
 
+  @Test(expected = AssertionError.class)
+  public void testSchemaTestBuilderSetInvalidBaselineValues() throws Exception {
+    final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0";
+
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+            .setMinorType(TypeProtos.MinorType.VARCHAR)
+            .setMode(TypeProtos.DataMode.REQUIRED)
+            .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+            .sqlQuery(query)
+            .schemaBaseLine(expectedSchema)
+            .baselineValues(new Object[0])
+            .build()
+            .run();
+  }
+
+  @Test(expected = AssertionError.class)
+  public void testSchemaTestBuilderSetInvalidBaselineRecords() throws Exception {
+    final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0";
+
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+        .setMinorType(TypeProtos.MinorType.VARCHAR)
+        .setMode(TypeProtos.DataMode.REQUIRED)
+        .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query)
+        .schemaBaseLine(expectedSchema)
+        .baselineRecords(new ArrayList<Map>())
+        .build()
+        .run();
+  }
+
+  @Test(expected = AssertionError.class)
+  public void testSchemaTestBuilderSetInvalidBaselineColumns() throws Exception {
+    final String query = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0";
+
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+            .setMinorType(TypeProtos.MinorType.VARCHAR)
+            .setMode(TypeProtos.DataMode.REQUIRED)
+            .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query)
+        .baselineColumns("col")
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+  }
+
   @Test
   public void testCSVVerification() throws Exception {
     testBuilder()

http://git-wip-us.apache.org/repos/asf/drill/blob/bd14e1ad/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
new file mode 100644
index 0000000..7d3f6d0
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.util.FileUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
+  @Test
+  public void testConcatWithMoreThanTwoArgs() throws Exception {
+    final String query = "select concat(r_name, r_name, r_name) as col \n" +
+        "from cp.`tpch/region.parquet` limit 0";
+
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+            .setMinorType(TypeProtos.MinorType.VARCHAR)
+            .setMode(TypeProtos.DataMode.REQUIRED)
+            .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+  }
+
+  @Test
+  public void testTrimOnlyOneArg() throws Exception {
+    final String query1 = "SELECT ltrim('drill') as col FROM (VALUES(1)) limit 0";
+    final String query2 = "SELECT rtrim('drill') as col FROM (VALUES(1)) limit 0";
+    final String query3 = "SELECT btrim('drill') as col FROM (VALUES(1)) limit 0";
+
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+        .setMinorType(TypeProtos.MinorType.VARCHAR)
+        .setMode(TypeProtos.DataMode.REQUIRED)
+        .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query1)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+
+    testBuilder()
+        .sqlQuery(query2)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+
+    testBuilder()
+        .sqlQuery(query3)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+  }
+
+  @Test
+  public void testExtract() throws Exception {
+    final String query = "select extract(second from time '02:30:45.100') as col \n" +
+        "from cp.`employee.json` limit 0";
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+        .setMinorType(TypeProtos.MinorType.FLOAT8)
+        .setMode(TypeProtos.DataMode.OPTIONAL)
+        .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+  }
+
+  @Test
+  public void tesIsNull() throws Exception {
+    final String query = "select r_name is null as col from cp.`tpch/region.parquet` limit
0";
+    List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
+    TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
+        .setMinorType(TypeProtos.MinorType.BIT)
+        .setMode(TypeProtos.DataMode.OPTIONAL)
+        .build();
+    expectedSchema.add(Pair.of(SchemaPath.getSimplePath("col"), majorType));
+
+    testBuilder()
+        .sqlQuery(query)
+        .schemaBaseLine(expectedSchema)
+        .build()
+        .run();
+  }
+}


Mime
View raw message