drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject drill git commit: DRILL-3567: Support multiple window definitions; Bump DrillCalcite version number to 1.1.0-drill-r16
Date Wed, 05 Aug 2015 17:15:59 GMT
Repository: drill
Updated Branches:
  refs/heads/master a96f793d7 -> 317b92e40


DRILL-3567: Support multiple window definitions; Bump DrillCalcite version number to 1.1.0-drill-r16


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/317b92e4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/317b92e4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/317b92e4

Branch: refs/heads/master
Commit: 317b92e409d62efe56fba1ecba944c13945d4f7b
Parents: a96f793
Author: Hsuan-Yi Chu <hsuanyi@usc.edu>
Authored: Sat Aug 1 12:04:15 2015 -0700
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Wed Aug 5 10:04:31 2015 -0700

----------------------------------------------------------------------
 .../sql/parser/UnsupportedOperatorsVisitor.java | 87 ++++++++++--------
 .../apache/drill/exec/TestWindowFunctions.java  | 92 +++++++++++---------
 pom.xml                                         |  2 +-
 3 files changed, 106 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/317b92e4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
index 17689ad..04d1231 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
@@ -35,7 +35,6 @@ import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.util.SqlShuttle;
 import org.apache.calcite.sql.SqlDataTypeSpec;
-import org.apache.calcite.sql.SqlSetOperator;
 
 import java.util.List;
 
@@ -88,9 +87,9 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
     if(sqlCall instanceof SqlSelect) {
       SqlSelect sqlSelect = (SqlSelect) sqlCall;
 
-      // This is used to keep track of the window function which has been defined
-      SqlNode definedWindow = null;
       for(SqlNode nodeInSelectList : sqlSelect.getSelectList()) {
+        // If the window function is used with an alias,
+        // enter the first operand of AS operator
         if(nodeInSelectList.getKind() == SqlKind.AS
             && (((SqlCall) nodeInSelectList).getOperandList().get(0).getKind() ==
SqlKind.OVER)) {
           nodeInSelectList = ((SqlCall) nodeInSelectList).getOperandList().get(0);
@@ -129,7 +128,6 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
                 break;
             }
 
-
             // DRILL-3182
             // Window function with DISTINCT qualifier is temporarily disabled
             if(function.getFunctionQuantifier() != null
@@ -140,36 +138,6 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
               throw new UnsupportedOperationException();
             }
           }
-
-          // DRILL-3196
-          // Disable multiple partitions in a SELECT-CLAUSE
-          SqlNode window = ((SqlCall) nodeInSelectList).operand(1);
-
-          // Partition window is referenced as a SqlIdentifier,
-          // which is defined in the window list
-          if(window instanceof SqlIdentifier) {
-            // Expand the SqlIdentifier as the expression defined in the window list
-            for(SqlNode sqlNode : sqlSelect.getWindowList()) {
-              if(((SqlWindow) sqlNode).getDeclName().equalsDeep(window, false)) {
-                window = sqlNode;
-                break;
-              }
-            }
-
-            assert !(window instanceof SqlIdentifier) : "Identifier should have been expanded
as a window defined in the window list";
-          }
-
-          // In a SELECT-SCOPE, only a partition can be defined
-          if(definedWindow == null) {
-            definedWindow = window;
-          } else {
-            if(!definedWindow.equalsDeep(window, false)) {
-              unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
-                  "Multiple window definitions in a single SELECT list is not currently supported
\n" +
-                  "See Apache Drill JIRA: DRILL-3196");
-              throw new UnsupportedOperationException();
-            }
-          }
         }
       }
     }
@@ -325,4 +293,55 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
     return sqlNode instanceof SqlCall
         && ((SqlCall) sqlNode).getOperator().getName().toLowerCase().equals("flatten");
   }
+
+  /**
+   * Disable multiple partitions in a SELECT-CLAUSE
+   * If multiple partitions are defined in the query,
+   * SqlUnsupportedException would be thrown to inform
+   * @param sqlSelect SELECT-CLAUSE in the query
+   */
+  private void detectMultiplePartitions(SqlSelect sqlSelect) {
+    for(SqlNode nodeInSelectList : sqlSelect.getSelectList()) {
+      // If the window function is used with an alias,
+      // enter the first operand of AS operator
+      if(nodeInSelectList.getKind() == SqlKind.AS
+          && (((SqlCall) nodeInSelectList).getOperandList().get(0).getKind() == SqlKind.OVER))
{
+        nodeInSelectList = ((SqlCall) nodeInSelectList).getOperandList().get(0);
+      }
+
+      if(nodeInSelectList.getKind() != SqlKind.OVER) {
+        continue;
+      }
+
+      // This is used to keep track of the window function which has been defined
+      SqlNode definedWindow = null;
+      SqlNode window = ((SqlCall) nodeInSelectList).operand(1);
+
+      // Partition window is referenced as a SqlIdentifier,
+      // which is defined in the window list
+      if(window instanceof SqlIdentifier) {
+        // Expand the SqlIdentifier as the expression defined in the window list
+        for(SqlNode sqlNode : sqlSelect.getWindowList()) {
+          if(((SqlWindow) sqlNode).getDeclName().equalsDeep(window, false)) {
+            window = sqlNode;
+            break;
+          }
+        }
+
+        assert !(window instanceof SqlIdentifier) : "Identifier should have been expanded
as a window defined in the window list";
+      }
+
+      // In a SELECT-SCOPE, only a partition can be defined
+      if(definedWindow == null) {
+        definedWindow = window;
+      } else {
+        if(!definedWindow.equalsDeep(window, false)) {
+          unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
+              "Multiple window definitions in a single SELECT list is not currently supported
\n" +
+              "See Apache Drill JIRA: DRILL-3196");
+          throw new UnsupportedOperationException();
+        }
+      }
+    }
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/317b92e4/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
index 222c7b7..ce5a477 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
@@ -51,46 +51,6 @@ public class TestWindowFunctions extends BaseTestQuery {
     test(query);
   }
 
-  @Test(expected = UnsupportedFunctionException.class) // DRILL-3196
-  public void testMultiplePartitions() throws Exception {
-    try {
-      final String query = "explain plan for select sum(a2) over(partition by a2), count(*)
over(partition by a2,b2,c2) \n" +
-          "from cp.`tpch/nation.parquet`";
-
-      test(query);
-    } catch(UserException ex) {
-      throwAsUnsupportedException(ex);
-      throw ex;
-    }
-  }
-
-  @Test(expected = UnsupportedFunctionException.class) // DRILL-3196
-  public void testSinglePartitionMultipleOrderBy() throws Exception {
-    try {
-      final String query = "explain plan for select sum(a2) over(partition by a2 order by
a2), count(*) over(partition by a2 order by b2) \n" +
-          "from cp.`tpch/nation.parquet`";
-
-      test(query);
-    } catch(UserException ex) {
-      throwAsUnsupportedException(ex);
-      throw ex;
-    }
-  }
-
-  @Test(expected = UnsupportedFunctionException.class) // DRILL-3196
-  public void testMultiplePartitionsDefinedInWindowList() throws Exception {
-    try {
-      final String query = "explain plan for select sum(a2) over(partition by a2), count(*)
over w \n" +
-          "from cp.`tpch/nation.parquet` \n" +
-          "window w as (partition by a2, b2, c2)";
-
-      test(query);
-    } catch(UserException ex) {
-      throwAsUnsupportedException(ex);
-      throw ex;
-    }
-  }
-
   @Test(expected = UnsupportedFunctionException.class) // DRILL-3182
   public void testWindowFunctionWithDistinct() throws Exception {
     try {
@@ -438,4 +398,56 @@ public class TestWindowFunctions extends BaseTestQuery {
         .build()
         .run();
   }
+
+  @Test // DRILL-3567
+  public void testMultiplePartitions1() throws Exception {
+    String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+    String query = String.format("select count(*) over(partition by b1 order by c1) as count1,
\n" +
+        "sum(a1)  over(partition by b1 order by c1) as sum1, \n" +
+        "count(*) over(partition by a1 order by c1) as count2 \n" +
+        "from dfs_test.`%s`", root);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .baselineColumns("count1", "sum1", "count2")
+        .baselineValues(1l, 0l, 2l)
+        .baselineValues(1l, 0l, 2l)
+        .baselineValues(2l, 0l, 5l)
+        .baselineValues(3l, 0l, 5l)
+        .baselineValues(3l, 0l, 5l)
+        .baselineValues(1l, 10l, 2l)
+        .baselineValues(1l, 10l, 2l)
+        .baselineValues(2l, 20l, 5l)
+        .baselineValues(3l, 30l, 5l)
+        .baselineValues(3l, 30l, 5l)
+        .build()
+        .run();
+  }
+
+  @Test // DRILL-3567
+  public void testMultiplePartitions2() throws Exception {
+    String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+    String query = String.format("select count(*) over(partition by b1 order by c1) as count1,
\n" +
+        "count(*) over(partition by a1 order by c1) as count2, \n" +
+        "sum(a1)  over(partition by b1 order by c1) as sum1 \n" +
+        "from dfs_test.`%s`", root);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .baselineColumns("count1", "count2", "sum1")
+        .baselineValues(1l, 2l, 0l)
+        .baselineValues(1l, 2l, 0l)
+        .baselineValues(2l, 5l, 0l)
+        .baselineValues(3l, 5l, 0l)
+        .baselineValues(3l, 5l, 0l)
+        .baselineValues(1l, 2l, 10l)
+        .baselineValues(1l, 2l, 10l)
+        .baselineValues(2l, 5l, 20l)
+        .baselineValues(3l, 5l, 30l)
+        .baselineValues(3l, 5l, 30l)
+        .build()
+        .run();
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/317b92e4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3ec79e1..e95e7f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1226,7 +1226,7 @@
           <dependency>
             <groupId>org.apache.calcite</groupId>
             <artifactId>calcite-core</artifactId>
-            <version>1.1.0-drill-r15</version>
+            <version>1.1.0-drill-r16</version>
             <exclusions>
               <exclusion>
                 <groupId>org.jgrapht</groupId>


Mime
View raw message