drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject [1/7] drill git commit: DRILL-2385: Count on complex objects failed with missing function implementation - added MapHolder, ListHolder; - added testCountComplexObjects() unit test.
Date Thu, 23 Jun 2016 18:10:41 GMT
Repository: drill
Updated Branches:
  refs/heads/1.7.0 [created] 552129fa6


DRILL-2385: Count on complex objects failed with missing function implementation - added MapHolder,
ListHolder; - added testCountComplexObjects() unit test.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c5216a69
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c5216a69
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c5216a69

Branch: refs/heads/1.7.0
Commit: c5216a691c10fd4e90eda68c93a06aab7d036ec5
Parents: bd1d9c2
Author: Vitalii Diravka <vitalii.diravka@gmail.com>
Authored: Thu Jun 16 16:45:20 2016 +0000
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Wed Jun 22 15:07:48 2016 -0700

----------------------------------------------------------------------
 .../src/main/codegen/data/CountAggrTypes.tdd    | 21 +++++++++++
 .../exec/resolver/ResolverTypePrecedence.java   |  2 ++
 .../drill/exec/resolver/TypeCastRules.java      |  2 ++
 .../exec/fn/impl/TestAggregateFunctions.java    | 34 ++++++++++++++++++
 .../test/resources/complex/json/complex.json    | 30 ++++++++++++++++
 .../resources/complex/json/repeated_list.json   |  2 +-
 .../complex/json/repeated_list_map.json         |  2 +-
 .../drill/exec/expr/holders/ListHolder.java     | 37 ++++++++++++++++++++
 .../drill/exec/expr/holders/MapHolder.java      | 32 +++++++++++++++++
 .../exec/expr/holders/RepeatedListHolder.java   | 20 +++++++++--
 .../exec/expr/holders/RepeatedMapHolder.java    | 25 +++++++++++--
 11 files changed, 201 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd b/exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd
index 53e25f7..aec73d1 100644
--- a/exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd
+++ b/exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd
@@ -2,38 +2,59 @@
   countFunctionsInput: [
     "Bit",
     "NullableBit",
+    "RepeatedBit",
     "Int",
     "NullableInt",
+    "RepeatedInt",
     "BigInt",
     "NullableBigInt",
+    "RepeatedBigInt",
     "Float4",
     "NullableFloat4",
+    "RepeatedFloat4",
     "Float8",
     "NullableFloat8",
+    "RepeatedFloat8",
     "Date",
     "NullableDate",
+    "RepeatedDate",
     "TimeStamp",
     "NullableTimeStamp",
+    "RepeatedTimeStamp",
     "Time",
     "NullableTime",
+    "RepeatedTime",
     "IntervalDay",
     "NullableIntervalDay",
+    "RepeatedIntervalDay",
     "IntervalYear",
     "NullableIntervalYear",
+    "RepeatedIntervalYear",
     "Interval",
     "NullableInterval",
+    "RepeatedInterval",
     "VarChar",
     "NullableVarChar",
+    "RepeatedVarChar",
     "VarBinary"
     "NullableVarBinary",
+    "RepeatedVarBinary"
     "Decimal9",
     "NullableDecimal9",
+    "RepeatedDecimal9",
     "Decimal18",
     "NullableDecimal18",
+    "RepeatedDecimal18",
     "Decimal28Sparse",
     "NullableDecimal28Sparse",
+    "RepeatedDecimal28Sparse",
     "Decimal38Sparse",
     "NullableDecimal38Sparse"
+    "RepeatedDecimal38Sparse",
+    "List"
+    "RepeatedList",
+    "Map"
+    "RepeatedMap"
   ]
 }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
index 8c602b3..a28c95a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
@@ -77,6 +77,8 @@ public class ResolverTypePrecedence {
     precedenceMap.put(MinorType.INTERVALDAY, i+= 2);
     precedenceMap.put(MinorType.INTERVALYEAR, i+= 2);
     precedenceMap.put(MinorType.INTERVAL, i+= 2);
+    precedenceMap.put(MinorType.MAP, i += 2);
+    precedenceMap.put(MinorType.LIST, i += 2);
     precedenceMap.put(MinorType.UNION, i += 2);
 
     MAX_IMPLICIT_CAST_COST = i;

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index ae42937..8bb6c2a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -754,6 +754,8 @@ public class TypeCastRules {
     rule.add(MinorType.FIXEDBINARY);
     rules.put(MinorType.VARBINARY, rule);
 
+    rules.put(MinorType.MAP, Sets.newHashSet(MinorType.MAP));
+    rules.put(MinorType.LIST, Sets.newHashSet(MinorType.LIST));
     rules.put(MinorType.UNION, Sets.newHashSet(MinorType.UNION));
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
index 0e558a7..d99eb00 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
@@ -18,6 +18,7 @@
 package org.apache.drill.exec.fn.impl;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.PlanTestBase;
@@ -28,6 +29,7 @@ import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.List;
+import java.util.Map;
 
 public class TestAggregateFunctions extends BaseTestQuery {
 
@@ -525,4 +527,36 @@ public class TestAggregateFunctions extends BaseTestQuery {
     PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPatterns);
   }
 
+  @Test // DRILL-2385: count on complex objects failed with missing function implementation
+  public void testCountComplexObjects() throws Exception {
+    final String query = "select count(t.%s) %s from cp.`complex/json/complex.json` t";
+    Map<String, String> objectsMap = Maps.newHashMap();
+    objectsMap.put("COUNT_BIG_INT_REPEATED", "sia");
+    objectsMap.put("COUNT_FLOAT_REPEATED", "sfa");
+    // TODO: can be uncommented after fixing DRILL-4664
+    // objectsMap.put("COUNT_MAP_REPEATED", "soa");
+    // objectsMap.put("COUNT_MAP_REQUIRED", "oooi");
+    objectsMap.put("COUNT_LIST_REPEATED", "odd");
+    objectsMap.put("COUNT_LIST_OPTIONAL", "sia");
+
+    for (String object: objectsMap.keySet()) {
+      String optionSetting = "";
+      if (object.equals("COUNT_LIST_OPTIONAL")) {
+        // if `exec.enable_union_type` parameter is true then BIGINT<REPEATED> object
is converted to LIST<OPTIONAL> one
+        optionSetting = "alter session set `exec.enable_union_type`=true";
+      }
+      try {
+        testBuilder()
+            .sqlQuery(query, objectsMap.get(object), object)
+            .optionSettingQueriesForTestQuery(optionSetting)
+            .unOrdered()
+            .baselineColumns(object)
+            .baselineValues(3L)
+            .go();
+      } finally {
+        test("ALTER SESSION RESET `exec.enable_union_type`");
+      }
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/test/resources/complex/json/complex.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/complex.json b/exec/java-exec/src/test/resources/complex/json/complex.json
new file mode 100644
index 0000000..5e52f63
--- /dev/null
+++ b/exec/java-exec/src/test/resources/complex/json/complex.json
@@ -0,0 +1,30 @@
+{"sia":[1, 11, 101, 1001],
+ "sfa":[0.0, 1.01, 10.222, 10.0006789],
+ "soa":[{"in":1},{"in":1,"fl":1.12345}, {"in":1, "fl":10.12345, "nul":null}, {"in":1, "fl":10.6789,
"nul":null, "bool":true, "str":"here is a string at row 1"}],
+ "oooi":{"oa":{"oab":{"oabc":1}}},
+ "odd": [
+    [[1],[],[3]],
+    [],
+    [[5]]
+ ]
+}
+{"sia":[2, 12, 102, 1002],
+ "sfa":[0.0, 2.01, 20.222, 20.0006789],
+ "soa":[{"in":2},{"in":2,"fl":2.12345}, {"in":2, "fl":20.12345, "nul":"not null"}, {"in":2,
"fl":20.6789, "nul":null, "bool":false, "str":"here is a string at row 2"}],
+ "oooi":{"oa":{"oab":{"oabc":2}}},
+ "odd": [
+    [[1],[],[3]],
+    [],
+    [[5]]
+ ]
+}
+{"sia":[3, 13, 103, 1003],
+ "sfa":[0.0, 3.01, 30.222, 30.0006789],
+ "soa":[{"in":3},{"in":3,"fl":3.12345}, {"in":3, "fl":30.12345, "nul":"not null"}, {"in":3,
"fl":30.6789, "nul":"not null", "bool":true, "str":"here is a string at row 3"}],
+ "oooi":{"oa":{"oab":{"oabc":3}}},
+ "odd": [
+    [[1],[],[3]],
+    [],
+    [[5]]
+ ]
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/test/resources/complex/json/repeated_list.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list.json b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
index 696d069..b3418f2 100644
--- a/exec/java-exec/src/test/resources/complex/json/repeated_list.json
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
@@ -4,4 +4,4 @@
     [],
     [[5]]
   ]
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
index ef6c0ee..cad3fe8 100644
--- a/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
@@ -4,4 +4,4 @@
     [],
     [{"val": [7]}]
   ]
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ListHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ListHolder.java
b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ListHolder.java
new file mode 100644
index 0000000..ebdbae9
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ListHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ListHolder implements ValueHolder {
+    public static final TypeProtos.MajorType TYPE = Types.optional(TypeProtos.MinorType.LIST);
+    public FieldReader reader;
+    public int isSet;
+
+    public TypeProtos.MajorType getType() {
+        return TYPE;
+    }
+
+    public boolean isSet() {
+        return isSet == 1;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/MapHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/MapHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/MapHolder.java
new file mode 100644
index 0000000..8a38bd4
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/MapHolder.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class MapHolder implements ValueHolder {
+    public static final TypeProtos.MajorType TYPE = Types.required(TypeProtos.MinorType.MAP);
+    public FieldReader reader;
+
+    public TypeProtos.MajorType getType() {
+        return TYPE;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
index 09746da..dc857de 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
@@ -17,7 +17,23 @@
  */
 package org.apache.drill.exec.expr.holders;
 
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.ListVector;
+
 public final class RepeatedListHolder implements ValueHolder{
-  public int start;
-  public int end;
+
+    public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.LIST);
+
+    public TypeProtos.MajorType getType() {return TYPE;}
+
+    /** The first index (inclusive) into the Vector. **/
+    public int start;
+
+    /** The last index (exclusive) into the Vector. **/
+    public int end;
+
+    /** The Vector holding the actual values. **/
+    public ListVector vector;
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/c5216a69/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
index 247f75e..3db9020 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
@@ -17,7 +17,28 @@
  */
 package org.apache.drill.exec.expr.holders;
 
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.complex.MapVector;
+
+import java.util.LinkedHashSet;
+
 public final class RepeatedMapHolder implements ValueHolder{
-  public int start;
-  public int end;
+
+    public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.MAP);
+
+//    public final LinkedHashSet<ValueHolder> children = null;
+
+    public TypeProtos.MajorType getType() {return TYPE;}
+
+    /** The first index (inclusive) into the Vector. **/
+    public int start;
+
+    /** The last index (exclusive) into the Vector. **/
+    public int end;
+
+    /** The Vector holding the actual values. **/
+    public MapVector vector;
+
 }


Mime
View raw message