tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [1/2] TAJO-801: Multiple distinct should be supported. (Hyoungjun Kim via hyunsik)
Date Thu, 15 May 2014 23:54:18 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 1c21e8bb6 -> 9350a8026


http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
index 629889d..4c4227f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
@@ -47,13 +47,13 @@ public class SortAggregateExec extends AggregationExec {
 
   public SortAggregateExec(TaskAttemptContext context, GroupbyNode plan, PhysicalExec child)
throws IOException {
     super(context, plan, child);
-    contexts = new FunctionContext[plan.getAggFunctions().length];
+    contexts = new FunctionContext[plan.getAggFunctions() == null ? 0 : plan.getAggFunctions().length];
   }
 
   @Override
   public Tuple next() throws IOException {
     Tuple currentKey;
-    Tuple tuple;
+    Tuple tuple = null;
     Tuple outputTuple = null;
 
     while(!context.isStopped() && (tuple = child.next()) != null) {
@@ -101,6 +101,10 @@ public class SortAggregateExec extends AggregationExec {
       }
     } // while loop
 
+    if (tuple == null && lastKey == null) {
+      finished = true;
+      return null;
+    }
     if (!finished) {
       outputTuple = new VTuple(outSchema.size());
       int tupleIdx = 0;

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index e3d3028..dbff67f 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -195,6 +195,7 @@ message EnforceProperty {
     SORT = 4;
     BROADCAST = 5;
     COLUMN_PARTITION = 6;
+    DISTINCT_GROUP_BY = 7;
   }
 
   // Identifies which field is filled in.
@@ -208,6 +209,7 @@ message EnforceProperty {
   optional SortEnforce sort = 6;
   optional BroadcastEnforce broadcast = 7;
   optional ColumnPartitionEnforcer columnPartition = 8;
+  optional DistinctGroupbyEnforcer distinct = 9;
 }
 
 message SortedInputEnforce {
@@ -266,6 +268,21 @@ message ColumnPartitionEnforcer {
   required ColumnPartitionAlgorithm algorithm = 2;
 }
 
+message DistinctGroupbyEnforcer {
+  enum DistinctAggregationAlgorithm {
+    HASH_AGGREGATION = 0;
+    SORT_AGGREGATION = 1;
+  }
+
+  message SortSpecArray {
+    required int32 pid = 1;
+    repeated SortSpecProto sortSpecs = 2;
+  }
+  required int32 pid = 1;
+  required DistinctAggregationAlgorithm algorithm = 2;
+  repeated SortSpecArray sortSpecArrays = 3;
+}
+
 message EnforcerProto {
   repeated EnforceProperty properties = 1;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index 1f8cb2a..91993a1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -179,7 +179,7 @@ public class TestGroupByQuery extends QueryTestCaseBase {
 
   @Test
   public final void testDistinctAggregation6() throws Exception {
-    // select count(distinct l_orderkey), sum(l_orderkey), sum(l_linenumber), count(*) as
v4 from lineitem
+    // select count(distinct l_orderkey) v0, sum(l_orderkey) v1, sum(l_linenumber) v2, count(*)
as v4 from lineitem
     // group by l_orderkey;
     ResultSet res = executeQuery();
     assertResultSet(res);
@@ -187,6 +187,15 @@ public class TestGroupByQuery extends QueryTestCaseBase {
   }
 
   @Test
+  public final void testDistinctAggregation7() throws Exception {
+    // select count(*), count(distinct c_nationkey), count(distinct c_mktsegment) from customer
+    // tpch scale 1000: 15000000	25	5
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
   public final void testDistinctAggregationWithHaving1() throws Exception {
     // select l_linenumber, count(*), count(distinct l_orderkey), sum(distinct l_orderkey)
from lineitem
     // group by l_linenumber having sum(distinct l_orderkey) >= 6;
@@ -205,6 +214,44 @@ public class TestGroupByQuery extends QueryTestCaseBase {
   }
 
   @Test
+  public final void testDistinctAggregationCasebyCase() throws Exception {
+    ResultSet res;
+
+    // one groupby, distinct, aggregation
+    res = executeFile("testDistinctAggregation_case1.sql");
+    assertResultSet(res, "testDistinctAggregation_case1.result");
+    res.close();
+
+    // one groupby, two distinct, one aggregation
+    res = executeFile("testDistinctAggregation_case2.sql");
+    assertResultSet(res, "testDistinctAggregation_case2.result");
+    res.close();
+
+    // one groupby, two distinct, two aggregation(no alias)
+    res = executeFile("testDistinctAggregation_case3.sql");
+    assertResultSet(res, "testDistinctAggregation_case3.result");
+    res.close();
+
+    // two groupby, two distinct, two aggregation
+    res = executeFile("testDistinctAggregation_case4.sql");
+    assertResultSet(res, "testDistinctAggregation_case4.result");
+    res.close();
+
+    // two groupby, two distinct, two aggregation with subquery
+    res = executeFile("testDistinctAggregation_case5.sql");
+    assertResultSet(res, "testDistinctAggregation_case5.result");
+    res.close();
+
+    res = executeFile("testDistinctAggregation_case6.sql");
+    assertResultSet(res, "testDistinctAggregation_case6.result");
+    res.close();
+
+    res = executeFile("testDistinctAggregation_case7.sql");
+    assertResultSet(res, "testDistinctAggregation_case7.result");
+    res.close();
+  }
+
+  @Test
   public final void testComplexParameter() throws Exception {
     // select sum(l_extendedprice*l_discount) as revenue from lineitem;
     ResultSet res = executeQuery();

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation7.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation7.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation7.sql
new file mode 100644
index 0000000..75b3eea
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation7.sql
@@ -0,0 +1,6 @@
+select
+    count(*),
+    count(distinct c_nationkey),
+    count(distinct c_mktsegment)
+from
+    customer
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case1.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case1.sql
new file mode 100644
index 0000000..4f01904
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case1.sql
@@ -0,0 +1,7 @@
+select
+    sum(l_quantity) as quantity,
+    count(distinct l_suppkey) suppkey,
+    l_returnflag
+from
+    lineitem
+group by l_returnflag
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case2.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case2.sql
new file mode 100644
index 0000000..10f1970
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case2.sql
@@ -0,0 +1,8 @@
+select
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey) as suppkey,
+    l_returnflag
+from
+    lineitem
+group by l_returnflag
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case3.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case3.sql
new file mode 100644
index 0000000..e585d42
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case3.sql
@@ -0,0 +1,9 @@
+select
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey),
+    max(l_quantity),
+    l_returnflag
+from
+    lineitem
+group by l_returnflag
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case4.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case4.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case4.sql
new file mode 100644
index 0000000..e8f05de
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case4.sql
@@ -0,0 +1,10 @@
+select
+    l_orderkey,
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey),
+    max(l_quantity),
+    l_returnflag
+from
+    lineitem
+group by l_returnflag, l_orderkey
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case5.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case5.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case5.sql
new file mode 100644
index 0000000..9a4d4eb
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case5.sql
@@ -0,0 +1,10 @@
+select
+    l_orderkey,
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey),
+    max(l_quantity),
+    l_returnflag
+from
+    (select * from lineitem) as litem
+group by l_returnflag, l_orderkey
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case6.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case6.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case6.sql
new file mode 100644
index 0000000..0a30fe3
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case6.sql
@@ -0,0 +1,12 @@
+select * from (
+select
+    l_orderkey,
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey),
+    max(l_quantity),
+    l_returnflag
+from
+    lineitem
+group by l_returnflag, l_orderkey
+) a
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case7.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case7.sql
b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case7.sql
new file mode 100644
index 0000000..e08c829
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregation_case7.sql
@@ -0,0 +1,9 @@
+select
+    count(distinct l_orderkey),
+    sum(l_quantity) as quantity,
+    count(distinct l_partkey) as partkey,
+    count(distinct l_suppkey),
+    max(l_quantity)
+from
+    lineitem
+group by l_orderkey
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation7.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation7.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation7.result
new file mode 100644
index 0000000..4173b04
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation7.result
@@ -0,0 +1,3 @@
+?count,?count_1,?count_2
+-------------------------------
+5,5,4
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case1.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case1.result
new file mode 100644
index 0000000..9296346
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case1.result
@@ -0,0 +1,4 @@
+quantity,suppkey,l_returnflag
+-------------------------------
+91.0,3,N
+94.0,2,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case2.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case2.result
new file mode 100644
index 0000000..7af127f
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case2.result
@@ -0,0 +1,4 @@
+quantity,partkey,suppkey,l_returnflag
+-------------------------------
+91.0,2,3,N
+94.0,2,2,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case3.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case3.result
new file mode 100644
index 0000000..31905fa
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case3.result
@@ -0,0 +1,4 @@
+quantity,partkey,?count,?max_1,l_returnflag
+-------------------------------
+91.0,2,3,38.0,N
+94.0,2,2,49.0,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case4.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case4.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case4.result
new file mode 100644
index 0000000..8afda65
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case4.result
@@ -0,0 +1,5 @@
+l_orderkey,quantity,partkey,?count,?max_1,l_returnflag
+-------------------------------
+1,53.0,1,2,36.0,N
+2,38.0,1,1,38.0,N
+3,94.0,2,2,49.0,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case5.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case5.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case5.result
new file mode 100644
index 0000000..8afda65
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case5.result
@@ -0,0 +1,5 @@
+l_orderkey,quantity,partkey,?count,?max_1,l_returnflag
+-------------------------------
+1,53.0,1,2,36.0,N
+2,38.0,1,1,38.0,N
+3,94.0,2,2,49.0,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case6.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case6.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case6.result
new file mode 100644
index 0000000..8afda65
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case6.result
@@ -0,0 +1,5 @@
+l_orderkey,quantity,partkey,?count,?max_1,l_returnflag
+-------------------------------
+1,53.0,1,2,36.0,N
+2,38.0,1,1,38.0,N
+3,94.0,2,2,49.0,R
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/9350a802/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case7.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case7.result
b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case7.result
new file mode 100644
index 0000000..03cdf1e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregation_case7.result
@@ -0,0 +1,5 @@
+?count,quantity,partkey,?count_1,?max_2
+-------------------------------
+1,53.0,1,2,36.0
+1,38.0,1,1,38.0
+1,94.0,2,2,49.0
\ No newline at end of file


Mime
View raw message