calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [3/3] calcite git commit: [CALCITE-1200] Extend RelOptUtil.splitJoinCondition to handle IS NOT DISTINCT FROM (Venki Korukanti)
Date Sat, 30 Apr 2016 01:25:05 GMT
[CALCITE-1200] Extend RelOptUtil.splitJoinCondition to handle IS NOT DISTINCT FROM (Venki Korukanti)

Close apache/calcite#220


Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/3599cebb
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/3599cebb
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/3599cebb

Branch: refs/heads/master
Commit: 3599cebbcfe52a2ee1c146fe97f75b168800e4f9
Parents: da875a6
Author: vkorukanti <venki@dremio.com>
Authored: Thu Apr 14 01:15:02 2016 -0700
Committer: Julian Hyde <jhyde@apache.org>
Committed: Fri Apr 29 18:21:04 2016 -0700

----------------------------------------------------------------------
 .../org/apache/calcite/plan/RelOptUtil.java     | 115 +++++++++++++++++--
 .../org/apache/calcite/rel/core/JoinInfo.java   |   3 +-
 .../rel/rules/AggregateJoinTransposeRule.java   |   3 +-
 .../org/apache/calcite/plan/RelOptUtilTest.java | 114 ++++++++++++++++++
 4 files changed, 226 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/calcite/blob/3599cebb/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index a03a0fa..d471b52 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -783,6 +783,15 @@ public abstract class RelOptUtil {
    *                  equi-join keys
    * @param rightKeys The ordinals of the fields from the right input which
    *                  are equi-join keys
+   * @param filterNulls List of boolean values for each join key position
+   *                    indicating whether the operator filters out nulls or not.
+   *                    Value is true if the operator is EQUALS and false if the
+   *                    operator is IS NOT DISTINCT FROM (or an expanded version).
+   *                    If <code>filterNulls</code> is null, only join conditions
+   *                    with EQUALS operators are considered equi-join components.
+   *                    Rest (including IS NOT DISTINCT FROM) are returned in
+   *                    remaining join condition.
+   *
    * @return remaining join filters that are not equijoins; may return a
    * {@link RexLiteral} true, but never null
    */
@@ -791,14 +800,17 @@ public abstract class RelOptUtil {
       RelNode right,
       RexNode condition,
       List<Integer> leftKeys,
-      List<Integer> rightKeys) {
+      List<Integer> rightKeys,
+      List<Boolean> filterNulls) {
     final List<RexNode> nonEquiList = new ArrayList<>();
 
     splitJoinCondition(
+        left.getCluster().getRexBuilder(),
         left.getRowType().getFieldCount(),
         condition,
         leftKeys,
         rightKeys,
+        filterNulls,
         nonEquiList);
 
     return RexUtil.composeConjunction(
@@ -819,12 +831,15 @@ public abstract class RelOptUtil {
       RexNode condition) {
     final List<Integer> leftKeys = new ArrayList<>();
     final List<Integer> rightKeys = new ArrayList<>();
+    final List<Boolean> filterNulls = new ArrayList<>();
     final List<RexNode> nonEquiList = new ArrayList<>();
     splitJoinCondition(
+        left.getCluster().getRexBuilder(),
         left.getRowType().getFieldCount(),
         condition,
         leftKeys,
         rightKeys,
+        filterNulls,
         nonEquiList);
     return nonEquiList.size() == 0;
   }
@@ -985,7 +1000,7 @@ public abstract class RelOptUtil {
 
     if (condition instanceof RexCall) {
       RexCall call = (RexCall) condition;
-      if (call.getOperator() == SqlStdOperatorTable.AND) {
+      if (call.getKind() == SqlKind.AND) {
         for (RexNode operand : call.getOperands()) {
           splitJoinCondition(
               sysFieldList,
@@ -1007,6 +1022,7 @@ public abstract class RelOptUtil {
       List<RelDataTypeField> rightFields = null;
       boolean reverse = false;
 
+      call = collapseExpandedIsNotDistinctFromExpr(call, rexBuilder);
       SqlKind kind = call.getKind();
 
       // Only consider range operators if we haven't already seen one
@@ -1348,30 +1364,39 @@ public abstract class RelOptUtil {
   }
 
   private static void splitJoinCondition(
+      final RexBuilder rexBuilder,
       final int leftFieldCount,
       RexNode condition,
       List<Integer> leftKeys,
       List<Integer> rightKeys,
+      List<Boolean> filterNulls,
       List<RexNode> nonEquiList) {
     if (condition instanceof RexCall) {
       RexCall call = (RexCall) condition;
-      final SqlOperator operator = call.getOperator();
-      if (operator == SqlStdOperatorTable.AND) {
+      SqlKind kind = call.getKind();
+      if (kind == SqlKind.AND) {
         for (RexNode operand : call.getOperands()) {
           splitJoinCondition(
+              rexBuilder,
               leftFieldCount,
               operand,
               leftKeys,
               rightKeys,
+              filterNulls,
               nonEquiList);
         }
         return;
       }
 
+      if (filterNulls != null) {
+        call = collapseExpandedIsNotDistinctFromExpr(call, rexBuilder);
+        kind = call.getKind();
+      }
+
       // "=" and "IS NOT DISTINCT FROM" are the same except for how they
-      // treat nulls. TODO: record null treatment
-      if (operator == SqlStdOperatorTable.EQUALS
-          || operator == SqlStdOperatorTable.IS_NOT_DISTINCT_FROM) {
+      // treat nulls.
+      if (kind == SqlKind.EQUALS
+          || (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM)) {
         final List<RexNode> operands = call.getOperands();
         if ((operands.get(0) instanceof RexInputRef)
             && (operands.get(1) instanceof RexInputRef)) {
@@ -1398,6 +1423,9 @@ public abstract class RelOptUtil {
 
           leftKeys.add(leftField.getIndex());
           rightKeys.add(rightField.getIndex() - leftFieldCount);
+          if (filterNulls != null) {
+            filterNulls.add(kind == SqlKind.EQUALS);
+          }
           return;
         }
         // Arguments were not field references, one from each side, so
@@ -1412,6 +1440,79 @@ public abstract class RelOptUtil {
   }
 
   /**
+   * Helper method for
+   * {@link #splitJoinCondition(RexBuilder, int, RexNode, List, List, List, List)} and
+   * {@link #splitJoinCondition(List, List, RexNode, List, List, List, List)}.
+   *
+   * <p>If the given expr <code>call</code> is an expanded version of
+   * IS NOT DISTINCT FROM function call, collapse it and return a
+   * IS NOT DISTINCT FROM function call.
+   *
+   * <p>For example: {@code t1.key IS NOT DISTINCT FROM t2.key}
+   * can rewritten in expanded form as
+   * {@code t1.key = t2.key OR (t1.key IS NULL AND t2.key IS NULL)}.
+   *
+   * @param call       Function expression to try collapsing.
+   * @param rexBuilder {@link RexBuilder} instance to create new {@link RexCall} instances.
+   * @return If the given function is an expanded IS NOT DISTINCT FROM function call,
+   *         return a IS NOT DISTINCT FROM function call. Otherwise return the input
+   *         function call as it is.
+   */
+  private static RexCall collapseExpandedIsNotDistinctFromExpr(final RexCall call,
+      final RexBuilder rexBuilder) {
+    if (call.getKind() != SqlKind.OR || call.getOperands().size() != 2) {
+      return call;
+    }
+
+    final RexNode op0 = call.getOperands().get(0);
+    final RexNode op1 = call.getOperands().get(1);
+
+    if (!(op0 instanceof RexCall) || !(op1 instanceof RexCall)) {
+      return call;
+    }
+
+    RexCall opEqCall = (RexCall) op0;
+    RexCall opNullEqCall = (RexCall) op1;
+
+    if (opEqCall.getKind() == SqlKind.AND
+        && opNullEqCall.getKind() == SqlKind.EQUALS) {
+      RexCall temp = opEqCall;
+      opEqCall = opNullEqCall;
+      opNullEqCall = temp;
+    }
+
+    if (opNullEqCall.getKind() != SqlKind.AND
+        || opNullEqCall.getOperands().size() != 2
+        || opEqCall.getKind() != SqlKind.EQUALS) {
+      return call;
+    }
+
+    final RexNode op10 = opNullEqCall.getOperands().get(0);
+    final RexNode op11 = opNullEqCall.getOperands().get(1);
+    if (op10.getKind() != SqlKind.IS_NULL
+        || op11.getKind() != SqlKind.IS_NULL) {
+      return call;
+    }
+    final RexNode isNullInput0 = ((RexCall) op10).getOperands().get(0);
+    final RexNode isNullInput1 = ((RexCall) op11).getOperands().get(0);
+
+    final String isNullInput0Digest = isNullInput0.toString();
+    final String isNullInput1Digest = isNullInput1.toString();
+    final String equalsInput0Digest = opEqCall.getOperands().get(0).toString();
+    final String equalsInput1Digest = opEqCall.getOperands().get(1).toString();
+
+    if ((isNullInput0Digest.equals(equalsInput0Digest)
+            && isNullInput1Digest.equals(equalsInput1Digest))
+        || (isNullInput1Digest.equals(equalsInput0Digest)
+            && isNullInput0Digest.equals(equalsInput1Digest))) {
+      return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM,
+          ImmutableList.of(isNullInput0, isNullInput1));
+    }
+
+    return call;
+  }
+
+  /**
    * Adding projection to the inputs of a join to produce the required join
    * keys.
    *

http://git-wip-us.apache.org/repos/asf/calcite/blob/3599cebb/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java b/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java
index 933eb0b..e044913 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java
@@ -56,9 +56,10 @@ public abstract class JoinInfo {
   public static JoinInfo of(RelNode left, RelNode right, RexNode condition) {
     final List<Integer> leftKeys = new ArrayList<Integer>();
     final List<Integer> rightKeys = new ArrayList<Integer>();
+    final List<Boolean> filterNulls = new ArrayList<Boolean>();
     RexNode remaining =
         RelOptUtil.splitJoinCondition(left, right, condition, leftKeys,
-            rightKeys);
+            rightKeys, filterNulls);
     if (remaining.isAlwaysTrue()) {
       return new EquiJoinInfo(ImmutableIntList.copyOf(leftKeys),
           ImmutableIntList.copyOf(rightKeys));

http://git-wip-us.apache.org/repos/asf/calcite/blob/3599cebb/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java
index 6978e04..fa76306 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java
@@ -168,9 +168,10 @@ public class AggregateJoinTransposeRule extends RelOptRule {
     // Split join condition
     final List<Integer> leftKeys = Lists.newArrayList();
     final List<Integer> rightKeys = Lists.newArrayList();
+    final List<Boolean> filterNulls = Lists.newArrayList();
     RexNode nonEquiConj =
         RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(),
-            join.getCondition(), leftKeys, rightKeys);
+            join.getCondition(), leftKeys, rightKeys, filterNulls);
     // If it contains non-equi join conditions, we bail out
     if (!nonEquiConj.isAlwaysTrue()) {
       return;

http://git-wip-us.apache.org/repos/asf/calcite/blob/3599cebb/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java
index 4f9b8cd..03bd725 100644
--- a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java
+++ b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java
@@ -16,16 +16,32 @@
  */
 package org.apache.calcite.plan;
 
+import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParser;
 import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.test.CalciteAssert;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.util.TestUtil;
 import org.apache.calcite.util.Util;
 
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
 import org.junit.Test;
 
+import java.util.Collections;
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -33,6 +49,24 @@ import static org.junit.Assert.fail;
  * Unit test for {@link RelOptUtil} and other classes in this package.
  */
 public class RelOptUtilTest {
+  /** Creates a config based on the "scott" schema. */
+  private static Frameworks.ConfigBuilder config() {
+    final SchemaPlus rootSchema = Frameworks.createRootSchema(true);
+    return Frameworks.newConfigBuilder()
+        .parserConfig(SqlParser.Config.DEFAULT)
+        .defaultSchema(CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.SCOTT));
+  }
+
+  private static final RelBuilder REL_BUILDER = RelBuilder.create(config().build());
+  private static final RelNode EMP_SCAN = REL_BUILDER.scan("EMP").build();
+  private static final RelNode DEPT_SCAN = REL_BUILDER.scan("DEPT").build();
+
+  private static final RelDataType EMP_ROW = EMP_SCAN.getRowType();
+  private static final RelDataType DEPT_ROW = DEPT_SCAN.getRowType();
+
+  private static final List<RelDataTypeField> EMP_DEPT_JOIN_REL_FIELDS =
+      Lists.newArrayList(Iterables.concat(EMP_ROW.getFieldList(), DEPT_ROW.getFieldList()));
+
   //~ Constructors -----------------------------------------------------------
 
   public RelOptUtilTest() {
@@ -87,6 +121,86 @@ public class RelOptUtilTest {
           e.getMessage());
     }
   }
+
+  /**
+   * Test {@link RelOptUtil#splitJoinCondition(RelNode, RelNode, RexNode, List, List, List)}
+   * where the join condition contains just one which is a EQUAL operator.
+   */
+  @Test public void testSplitJoinConditionEquals() {
+    int leftJoinIndex = EMP_SCAN.getRowType().getFieldNames().indexOf("DEPTNO");
+    int rightJoinIndex = DEPT_ROW.getFieldNames().indexOf("DEPTNO");
+
+    RexNode joinCond = REL_BUILDER.call(SqlStdOperatorTable.EQUALS,
+        RexInputRef.of(leftJoinIndex, EMP_DEPT_JOIN_REL_FIELDS),
+        RexInputRef.of(EMP_ROW.getFieldCount() + rightJoinIndex, EMP_DEPT_JOIN_REL_FIELDS));
+
+    splitJoinConditionHelper(
+        joinCond,
+        Collections.singletonList(leftJoinIndex),
+        Collections.singletonList(rightJoinIndex),
+        Collections.singletonList(true),
+        REL_BUILDER.literal(true));
+  }
+
+  /**
+   * Test {@link RelOptUtil#splitJoinCondition(RelNode, RelNode, RexNode, List, List, List)}
+   * where the join condition contains just one which is a IS NOT DISTINCT operator.
+   */
+  @Test public void testSplitJoinConditionIsNotDistinctFrom() {
+    int leftJoinIndex = EMP_SCAN.getRowType().getFieldNames().indexOf("DEPTNO");
+    int rightJoinIndex = DEPT_ROW.getFieldNames().indexOf("DEPTNO");
+
+    RexNode joinCond = REL_BUILDER.call(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM,
+        RexInputRef.of(leftJoinIndex, EMP_DEPT_JOIN_REL_FIELDS),
+        RexInputRef.of(EMP_ROW.getFieldCount() + rightJoinIndex, EMP_DEPT_JOIN_REL_FIELDS));
+
+    splitJoinConditionHelper(
+        joinCond,
+        Collections.singletonList(leftJoinIndex),
+        Collections.singletonList(rightJoinIndex),
+        Collections.singletonList(false),
+        REL_BUILDER.literal(true));
+  }
+
+  /**
+   * Test {@link RelOptUtil#splitJoinCondition(RelNode, RelNode, RexNode, List, List, List)}
+   * where the join condition contains an expanded version of IS NOT DISTINCT
+   */
+  @Test public void testSplitJoinConditionExpandedIsNotDistinctFrom() {
+    int leftJoinIndex = EMP_SCAN.getRowType().getFieldNames().indexOf("DEPTNO");
+    int rightJoinIndex = DEPT_ROW.getFieldNames().indexOf("DEPTNO");
+
+    RexInputRef leftKeyInputRef = RexInputRef.of(leftJoinIndex, EMP_DEPT_JOIN_REL_FIELDS);
+    RexInputRef rightKeyInputRef =
+        RexInputRef.of(EMP_ROW.getFieldCount() + rightJoinIndex, EMP_DEPT_JOIN_REL_FIELDS);
+    RexNode joinCond = REL_BUILDER.call(SqlStdOperatorTable.OR,
+        REL_BUILDER.call(SqlStdOperatorTable.EQUALS, leftKeyInputRef, rightKeyInputRef),
+        REL_BUILDER.call(SqlStdOperatorTable.AND,
+            REL_BUILDER.call(SqlStdOperatorTable.IS_NULL, leftKeyInputRef),
+            REL_BUILDER.call(SqlStdOperatorTable.IS_NULL, rightKeyInputRef)));
+
+    splitJoinConditionHelper(
+        joinCond,
+        Collections.singletonList(leftJoinIndex),
+        Collections.singletonList(rightJoinIndex),
+        Collections.singletonList(false),
+        REL_BUILDER.literal(true));
+  }
+
+  private static void splitJoinConditionHelper(RexNode joinCond, List<Integer> expLeftKeys,
+      List<Integer> expRightKeys, List<Boolean> expFilterNulls, RexNode expRemaining)
{
+    List<Integer> actLeftKeys = Lists.newArrayList();
+    List<Integer> actRightKeys = Lists.newArrayList();
+    List<Boolean> actFilterNulls = Lists.newArrayList();
+
+    RexNode actRemaining = RelOptUtil.splitJoinCondition(EMP_SCAN, DEPT_SCAN, joinCond, actLeftKeys,
+        actRightKeys, actFilterNulls);
+
+    assertEquals(expRemaining.toString(), actRemaining.toString());
+    assertEquals(expFilterNulls, actFilterNulls);
+    assertEquals(expLeftKeys, actLeftKeys);
+    assertEquals(expRightKeys, actRightKeys);
+  }
 }
 
 // End RelOptUtilTest.java


Mime
View raw message