calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [1/2] calcite git commit: [CALCITE-2016] ITEM + DOT operator does not work for array (Shuyi Chen)
Date Thu, 16 Nov 2017 21:15:57 GMT
Repository: calcite
Updated Branches:
  refs/heads/master 051809b4d -> a3dd90f10


[CALCITE-2016] ITEM + DOT operator does not work for array (Shuyi Chen)

Add parser rules to support chaining ITEM and DOT operators for
complex types.

Close apache/calcite#557


Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/a3dd90f1
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/a3dd90f1
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/a3dd90f1

Branch: refs/heads/master
Commit: a3dd90f1088966dc09caecf88779853ba34d290c
Parents: e8481e5
Author: Shuyi Chen <shuyi@uber.com>
Authored: Sun Nov 5 23:09:02 2017 -0800
Committer: Julian Hyde <jhyde@apache.org>
Committed: Wed Nov 15 22:45:49 2017 -0800

----------------------------------------------------------------------
 core/src/main/codegen/templates/Parser.jj       |  10 ++
 .../apache/calcite/sql/fun/SqlDotOperator.java  | 169 +++++++++++++++++++
 .../calcite/sql/fun/SqlStdOperatorTable.java    |  10 +-
 .../calcite/sql/validate/SqlValidatorImpl.java  |  13 +-
 .../sql2rel/StandardConvertletTable.java        |  10 ++
 .../calcite/sql/parser/SqlParserTest.java       |   5 +
 .../apache/calcite/test/MockCatalogReader.java  |   9 +-
 .../calcite/test/SqlToRelConverterTest.java     |   4 +
 .../apache/calcite/test/SqlValidatorTest.java   |  15 +-
 .../calcite/test/SqlToRelConverterTest.xml      |  11 ++
 10 files changed, 241 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/codegen/templates/Parser.jj
----------------------------------------------------------------------
diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj
index fb0439b..f71abce 100644
--- a/core/src/main/codegen/templates/Parser.jj
+++ b/core/src/main/codegen/templates/Parser.jj
@@ -2974,6 +2974,7 @@ List<Object> Expression2(ExprContext exprContext) :
     SqlNodeList nodeList;
     SqlNode e;
     SqlOperator op;
+    String p;
     final Span s = span();
 }
 {
@@ -3095,6 +3096,15 @@ List<Object> Expression2(ExprContext exprContext) :
                             SqlStdOperatorTable.ITEM, getPos()));
                     list.add(e);
                 }
+                (
+                    <DOT>
+                    p = Identifier() {
+                        list.add(
+                            new SqlParserUtil.ToTreeListItem(
+                                SqlStdOperatorTable.DOT, getPos()));
+                        list.add(new SqlIdentifier(p, getPos()));
+                    }
+                )*
             |
                 {
                     checkNonQueryExpression(exprContext);

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
new file mode 100644
index 0000000..45e9bd9
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperandCountRange;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.SqlSpecialOperator;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.util.Static;
+
+import java.util.Arrays;
+
+/**
+ * The dot operator {@code .}, used to access a field of a
+ * record. For example, {@code a.b}.
+ */
+public class SqlDotOperator extends SqlSpecialOperator {
+  SqlDotOperator() {
+    super("DOT", SqlKind.DOT, 100, true, null, null, null);
+  }
+
+  @Override public ReduceResult reduceExpr(int ordinal, TokenSequence list) {
+    SqlNode left = list.node(ordinal - 1);
+    SqlNode right = list.node(ordinal + 1);
+    return new ReduceResult(ordinal - 1,
+        ordinal + 2,
+        createCall(
+            SqlParserPos.sum(
+                Arrays.asList(left.getParserPosition(),
+                    right.getParserPosition(),
+                    list.pos(ordinal))),
+            left,
+            right));
+  }
+
+  @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec,
+      int rightPrec) {
+    final SqlWriter.Frame frame =
+        writer.startList(SqlWriter.FrameTypeEnum.IDENTIFIER);
+    call.operand(0).unparse(writer, leftPrec, 0);
+    writer.sep(".");
+    call.operand(1).unparse(writer, 0, 0);
+    writer.endList(frame);
+  }
+
+  @Override public SqlOperandCountRange getOperandCountRange() {
+    return SqlOperandCountRanges.of(2);
+  }
+
+  @Override public <R> void acceptCall(SqlVisitor<R> visitor, SqlCall call,
+      boolean onlyExpressions, SqlBasicVisitor.ArgHandler<R> argHandler) {
+    // Do not visit operands[1] here.
+    argHandler.visitChild(visitor, call, 0, call.operand(0));
+  }
+
+  @Override public RelDataType deriveType(SqlValidator validator,
+      SqlValidatorScope scope, SqlCall call) {
+    RelDataType nodeType = validator.deriveType(scope, call.getOperandList().get(0));
+    assert nodeType != null;
+
+    final String fieldName = call.getOperandList().get(1).toString();
+    RelDataTypeField field =
+        nodeType.getField(fieldName, false, false);
+    if (field == null) {
+      throw SqlUtil.newContextException(SqlParserPos.ZERO, Static.RESOURCE.unknownField(fieldName));
+    }
+    RelDataType type = field.getType();
+
+    // Validate and determine coercibility and resulting collation
+    // name of binary operator if needed.
+    type = adjustType(validator, call, type);
+    SqlValidatorUtil.checkCharsetAndCollateConsistentIfCharType(type);
+    return type;
+  }
+
+  public void validateCall(
+      SqlCall call,
+      SqlValidator validator,
+      SqlValidatorScope scope,
+      SqlValidatorScope operandScope) {
+    assert call.getOperator() == this;
+    // Do not visit call.getOperandList().get(1) here.
+    // call.getOperandList().get(1) will be validated when deriveType() is called.
+    call.getOperandList().get(0).validateExpr(validator, operandScope);
+  }
+
+  @Override public boolean checkOperandTypes(SqlCallBinding callBinding,
+      boolean throwOnFailure) {
+    final SqlNode left = callBinding.operand(0);
+    final SqlNode right = callBinding.operand(1);
+    final RelDataType type =
+        callBinding.getValidator().deriveType(callBinding.getScope(), left);
+    if (type.getSqlTypeName() != SqlTypeName.ROW) {
+      return false;
+    }
+    final RelDataType operandType = callBinding.getOperandType(0);
+    final SqlSingleOperandTypeChecker checker = getChecker(operandType);
+    return checker.checkSingleOperandType(callBinding, right, 0,
+        throwOnFailure);
+  }
+
+  private SqlSingleOperandTypeChecker getChecker(RelDataType operandType) {
+    switch (operandType.getSqlTypeName()) {
+    case ROW:
+      return OperandTypes.family(SqlTypeFamily.STRING);
+    default:
+      throw new AssertionError(operandType.getSqlTypeName());
+    }
+  }
+
+  @Override public String getAllowedSignatures(String name) {
+    return "<A>.<B>";
+  }
+
+  @Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+    final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
+    final RelDataType recordType = opBinding.getOperandType(0);
+    switch (recordType.getSqlTypeName()) {
+    case ROW:
+      final String fieldName =
+          opBinding.getOperandLiteralValue(1, String.class);
+      final RelDataType type = opBinding.getOperandType(0)
+          .getField(fieldName, false, false)
+          .getType();
+      if (recordType.isNullable()) {
+        return typeFactory.createTypeWithNullability(type, true);
+      } else {
+        return type;
+      }
+    default:
+      throw new AssertionError();
+    }
+  }
+}
+
+// End SqlDotOperator.java

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
index f6a762b..c56b8bc 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
@@ -295,15 +295,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
   /**
    * Dot operator, '<code>.</code>', used for referencing fields of records.
    */
-  public static final SqlBinaryOperator DOT =
-      new SqlBinaryOperator(
-          ".",
-          SqlKind.DOT,
-          80,
-          true,
-          null,
-          null,
-          OperandTypes.ANY_ANY);
+  public static final SqlOperator DOT = new SqlDotOperator();
 
   /**
    * Logical equals operator, '<code>=</code>'.

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 90bdb88..940c8ac 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -3273,6 +3273,13 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
     }
   }
 
+  private SqlNode stripDot(SqlNode node) {
+    if (node != null && node.getKind() == SqlKind.DOT) {
+      return ((SqlCall) node).operand(0);
+    }
+    return node;
+  }
+
   private void checkRollUp(SqlNode grandParent, SqlNode parent,
                            SqlNode current, SqlValidatorScope scope, String optionalClause)
{
     current = stripAs(current);
@@ -3281,7 +3288,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
       checkRollUpInWindow(getWindowInOver(current), scope);
       current = stripOver(current);
 
-      List<SqlNode> children = ((SqlCall) current).getOperandList();
+      List<SqlNode> children = ((SqlCall) stripDot(current)).getOperandList();
       for (SqlNode child : children) {
         checkRollUp(parent, current, child, scope, optionalClause);
       }
@@ -3289,10 +3296,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
       SqlIdentifier id = (SqlIdentifier) current;
       if (!id.isStar() && isRolledUpColumn(id, scope)) {
         if (!isAggregation(parent.getKind())
-                || !isRolledUpColumnAllowedInAgg(id, scope, (SqlCall) parent, grandParent))
{
+            || !isRolledUpColumnAllowedInAgg(id, scope, (SqlCall) parent, grandParent)) {
           String context = optionalClause != null ? optionalClause : parent.getKind().toString();
           throw newValidationError(id,
-                  RESOURCE.rolledUpNotAllowed(deriveAlias(id, 0), context));
+              RESOURCE.rolledUpNotAllowed(deriveAlias(id, 0), context));
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
index 4c45716..158ffa6 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
@@ -248,6 +248,16 @@ public class StandardConvertletTable extends ReflectiveConvertletTable
{
           }
         });
 
+    // "DOT"
+    registerOp(
+        SqlStdOperatorTable.DOT,
+        new SqlRexConvertlet() {
+          public RexNode convertCall(SqlRexContext cx, SqlCall call) {
+            return cx.getRexBuilder().makeCall(SqlStdOperatorTable.DOT,
+                cx.convertExpression(call.operand(0)),
+                cx.getRexBuilder().makeLiteral(call.operand(1).toString()));
+          }
+        });
     // "AS" has no effect, so expand "x AS id" into "x".
     registerOp(
         SqlStdOperatorTable.AS,

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
index 1042e20..4f2d9cb 100644
--- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -4250,6 +4250,11 @@ public class SqlParserTest {
     checkExp("a[b[1 + 2] + 3]", "`A`[(`B`[(1 + 2)] + 3)]");
   }
 
+  @Test public void testArrayElementWithDot() {
+    checkExp("a[1+2].b.c[2].d", "(((`A`[(1 + 2)].`B`).`C`)[2].`D`)");
+    checkExp("a[b[1]].c.f0[d[1]]", "((`A`[`B`[1]].`C`).`F0`)[`D`[1]]");
+  }
+
   @Test public void testArrayValueConstructor() {
     checkExp("array[1, 2]", "(ARRAY[1, 2])");
     checkExp("array [1, 2]", "(ARRAY[1, 2])"); // with space

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java b/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java
index ed1f952..b25ff10 100644
--- a/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java
+++ b/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java
@@ -1615,10 +1615,17 @@ public class MockCatalogReader extends CalciteCatalogReader {
                     .build())
             .kind(StructKind.PEEK_FIELDS_NO_EXPAND)
             .build();
+    final RelDataType skillRecordType =
+        typeFactory.builder()
+            .add("TYPE", varchar10Type)
+            .add("DESC", varchar20Type)
+            .build();
     final RelDataType empRecordType =
         typeFactory.builder()
             .add("EMPNO", intType)
-            .add("ENAME", varchar10Type).build();
+            .add("ENAME", varchar10Type)
+            .add("SKILLS", typeFactory.createArrayType(skillRecordType, -1))
+            .build();
     final RelDataType empListType =
         typeFactory.createArrayType(empRecordType, -1);
 

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
index aebcece..dd46878 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
@@ -1033,6 +1033,10 @@ public class SqlToRelConverterTest extends SqlToRelTestBase {
     sql(sql).ok();
   }
 
+  @Test public void testArrayOfRecord() {
+    sql("select employees[1].skills[2+3].desc from dept_nested").ok();
+  }
+
   @Test public void testUnnestArray() {
     sql("select*from unnest(array(select*from dept))").ok();
   }

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
index 570c7a0..49de12d 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
@@ -7400,7 +7400,9 @@ public class SqlValidatorTest extends SqlValidatorTestCase {
         + " UNNEST(d.employees) as e";
     final String type = "RecordType(INTEGER NOT NULL DEPTNO,"
         + " INTEGER NOT NULL EMPNO,"
-        + " VARCHAR(10) NOT NULL ENAME) NOT NULL";
+        + " VARCHAR(10) NOT NULL ENAME,"
+        + " RecordType(VARCHAR(10) NOT NULL TYPE, VARCHAR(20) NOT NULL DESC)"
+        + " NOT NULL ARRAY NOT NULL SKILLS) NOT NULL";
     sql(sql).type(type);
 
     // equivalent query using CROSS JOIN
@@ -7777,6 +7779,15 @@ public class SqlValidatorTest extends SqlValidatorTestCase {
         "RecordType(INTEGER NOT NULL X, VARCHAR(20) NOT NULL EMAIL, INTEGER NOT NULL Y) NOT
NULL");
   }
 
+  @Test public void testArrayOfRecordType() {
+    sql("SELECT name, dept_nested.employees[1].ne as ne from dept_nested")
+        .fails("Unknown field 'NE'");
+    sql("SELECT name, dept_nested.employees[1].ename as ename from dept_nested")
+        .type("RecordType(VARCHAR(10) NOT NULL NAME, VARCHAR(10) ENAME) NOT NULL");
+    sql("SELECT dept_nested.employees[1].skills[1].desc as DESCRIPTION from dept_nested")
+        .type("RecordType(VARCHAR(20) DESCRIPTION) NOT NULL");
+  }
+
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-497">[CALCITE-497]
    * Support optional qualifier for column name references</a>. */
@@ -8465,6 +8476,7 @@ public class SqlValidatorTest extends SqlValidatorTestCase {
         + "\n"
         + "CURRENT_VALUE -\n"
         + "DEFAULT -\n"
+        + "DOT -\n"
         + "ITEM -\n"
         + "NEXT_VALUE -\n"
         + "PATTERN_EXCLUDE -\n"
@@ -8476,7 +8488,6 @@ public class SqlValidatorTest extends SqlValidatorTestCase {
         + "$LiteralChain -\n"
         + "+ pre\n"
         + "- pre\n"
-        + ". left\n"
         + "FINAL pre\n"
         + "RUNNING pre\n"
         + "\n"

http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
index 6818e63..34037ba 100644
--- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
@@ -359,6 +359,17 @@ from orders
 window w as (partition by productId)]]>
         </Resource>
     </TestCase>
+    <TestCase name="testArrayOfRecord">
+        <Resource name="sql">
+            <![CDATA[select*from unnest(array(select*from dept))]]>
+        </Resource>
+        <Resource name="plan">
+            <![CDATA[
+LogicalProject(EXPR$0=[DOT(ITEM(DOT(ITEM($2, 1), 'SKILLS'), +(2, 3)), 'DESC')])
+  LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED]])
+]]>
+        </Resource>
+    </TestCase>
     <TestCase name="testUnnestArray">
         <Resource name="sql">
             <![CDATA[select*from unnest(array(select*from dept))]]>


Mime
View raw message