calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [25/27] calcite git commit: [CALCITE-842] Decorrelator gets field offsets confused if fields have been trimmed
Date Mon, 11 Jan 2016 04:47:26 GMT
[CALCITE-842] Decorrelator gets field offsets confused if fields have been trimmed


Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/4b519b98
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/4b519b98
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/4b519b98

Branch: refs/heads/master
Commit: 4b519b9882c861bf366e2c9d9928cd6deb5cc8b9
Parents: 505a906
Author: Julian Hyde <jhyde@apache.org>
Authored: Thu Jan 7 00:06:19 2016 -0800
Committer: Julian Hyde <jhyde@apache.org>
Committed: Sun Jan 10 00:51:25 2016 -0800

----------------------------------------------------------------------
 .../apache/calcite/rex/RexCorrelVariable.java   |   4 +-
 .../sql2rel/CorrelationReferenceFinder.java     |  74 +++++++++++
 .../apache/calcite/sql2rel/RelDecorrelator.java |  33 ++---
 .../apache/calcite/sql2rel/RelFieldTrimmer.java | 133 ++++++++++++-------
 .../calcite/sql2rel/SqlToRelConverter.java      |  20 +--
 .../apache/calcite/test/JdbcAdapterTest.java    |  93 ++++++-------
 .../java/org/apache/calcite/test/JdbcTest.java  |   8 +-
 .../org/apache/calcite/test/LatticeTest.java    |   2 +-
 .../apache/calcite/test/SqlToRelTestBase.java   |   2 +-
 .../enumerable/EnumerableCorrelateTest.java     |   5 +-
 core/src/test/resources/sql/subquery.iq         |   6 +-
 11 files changed, 236 insertions(+), 144 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
index 4880c8e..2f6197a 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
@@ -20,6 +20,8 @@ import org.apache.calcite.rel.core.CorrelationId;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.sql.SqlKind;
 
+import com.google.common.base.Preconditions;
+
 /**
  * Reference to the current row of a correlating relational expression.
  *
@@ -36,7 +38,7 @@ public class RexCorrelVariable extends RexVariable {
       CorrelationId id,
       RelDataType type) {
     super(id.getName(), type);
-    this.id = id;
+    this.id = Preconditions.checkNotNull(id);
   }
 
   //~ Methods ----------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java
b/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java
new file mode 100644
index 0000000..db84b4a
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql2rel;
+
+import org.apache.calcite.rel.RelHomogeneousShuttle;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
+
+/**
+ * Shuttle that finds references to a given {@link CorrelationId} within a tree
+ * of {@link RelNode}s.
+ */
+public abstract class CorrelationReferenceFinder extends RelHomogeneousShuttle {
+  private final MyRexVisitor rexVisitor;
+
+  /** Creates CorrelationReferenceFinder. */
+  protected CorrelationReferenceFinder() {
+    rexVisitor = new MyRexVisitor(this);
+  }
+
+  protected abstract RexNode handle(RexFieldAccess fieldAccess);
+
+  @Override public RelNode visit(RelNode other) {
+    RelNode next = super.visit(other);
+    return next.accept(rexVisitor);
+  }
+
+  /**
+   * Replaces alternative names of correlation variable to its canonical name.
+   */
+  private static class MyRexVisitor extends RexShuttle {
+    private final CorrelationReferenceFinder finder;
+
+    private MyRexVisitor(CorrelationReferenceFinder finder) {
+      this.finder = finder;
+    }
+
+    @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+      if (fieldAccess.getReferenceExpr() instanceof RexCorrelVariable) {
+        return finder.handle(fieldAccess);
+      }
+      return super.visitFieldAccess(fieldAccess);
+    }
+
+    @Override public RexNode visitSubQuery(RexSubQuery subQuery) {
+      final RelNode r = subQuery.rel.accept(finder); // look inside sub-queries
+      if (r != subQuery.rel) {
+        subQuery = subQuery.clone(r);
+      }
+      return super.visitSubQuery(subQuery);
+    }
+  }
+}
+
+// End CorrelationReferenceFinder.java

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 2812851..e1d9f93 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -701,8 +701,8 @@ public class RelDecorrelator implements ReflectiveVisitor {
 
     final Map<RelNode, Integer> mapNewInputToNewOffset = new HashMap<>();
 
-    // inputRel provides the definition of a correlated variable.
-    // Add to map all the referenced positions(relative to each input rel)
+    // Input provides the definition of a correlated variable.
+    // Add to map all the referenced positions (relative to each input rel).
     for (Correlation corVar : correlations) {
       final int oldCorVarOffset = corVar.field;
 
@@ -2365,14 +2365,15 @@ public class RelDecorrelator implements ReflectiveVisitor {
     }
 
     public int compareTo(Correlation o) {
-      int res = corr.compareTo(o.corr);
-      if (res != 0) {
-        return res;
+      int c = corr.compareTo(o.corr);
+      if (c != 0) {
+        return c;
       }
-      if (field != o.field) {
-        return field - o.field;
+      c = Integer.compare(field, o.field);
+      if (c != 0) {
+        return c;
       }
-      return uniqueKey - o.uniqueKey;
+      return Integer.compare(uniqueKey, o.uniqueKey);
     }
   }
 
@@ -2545,26 +2546,10 @@ public class RelDecorrelator implements ReflectiveVisitor {
                     corrIdGenerator++);
             mapFieldAccessToCorVar.put(fieldAccess, correlation);
             mapRefRelToCorVar.put(rel, correlation);
-/*
-            if (!mapCorVarToCorRel.containsKey(var.id)) {
-              mapCorVarToCorRel.put(var.id, Stacks.peek(stack));
-            }
-*/
           }
           return super.visitFieldAccess(fieldAccess);
         }
 
-        //@ Override
-        public Void visitCorrelVariable_(RexCorrelVariable var) {
-          final Correlation correlation =
-              new Correlation(var.id, -1, corrIdGenerator++);
-          mapRefRelToCorVar.put(rel, correlation);
-          if (!mapCorVarToCorRel.containsKey(var.id)) {
-            mapCorVarToCorRel.put(var.id, Stacks.peek(stack));
-          }
-          return super.visitCorrelVariable(var);
-        }
-
         @Override public Void visitSubQuery(RexSubQuery subQuery) {
           subQuery.rel.accept(CorelMapBuilder.this);
           return super.visitSubQuery(subQuery);

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
index 2294c4a..8638df0 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.CorrelationId;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.Project;
@@ -37,9 +38,12 @@ import org.apache.calcite.rel.logical.LogicalTableModify;
 import org.apache.calcite.rel.logical.LogicalValues;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rel.type.RelDataTypeImpl;
 import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexPermuteInputsShuttle;
@@ -66,8 +70,10 @@ import com.google.common.collect.Iterables;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
 
@@ -101,6 +107,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
 
   private final ReflectUtil.MethodDispatcher<TrimResult> trimFieldsDispatcher;
   private final RelBuilder relBuilder;
+  private Map<RelNode, Mapping> map = new HashMap<>();
 
   //~ Constructors -----------------------------------------------------------
 
@@ -177,23 +184,36 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
   protected TrimResult trimChild(
       RelNode rel,
       RelNode input,
-      ImmutableBitSet fieldsUsed,
+      final ImmutableBitSet fieldsUsed,
       Set<RelDataTypeField> extraFields) {
-    Util.discard(rel);
-    if (input.getClass().getName().endsWith("MedMdrClassExtentRel")) {
-      // MedMdrJoinRule cannot handle Join of Project of
-      // MedMdrClassExtentRel, only naked MedMdrClassExtentRel.
-      // So, disable trimming.
-      fieldsUsed = ImmutableBitSet.range(input.getRowType().getFieldCount());
-    }
+    final ImmutableBitSet.Builder fieldsUsedBuilder = fieldsUsed.rebuild();
+
+    // Fields that define the collation cannot be discarded.
     final ImmutableList<RelCollation> collations =
         RelMetadataQuery.collations(input);
     for (RelCollation collation : collations) {
       for (RelFieldCollation fieldCollation : collation.getFieldCollations()) {
-        fieldsUsed = fieldsUsed.set(fieldCollation.getFieldIndex());
+        fieldsUsedBuilder.set(fieldCollation.getFieldIndex());
       }
     }
-    return dispatchTrimFields(input, fieldsUsed, extraFields);
+
+    // Correlating variables are a means for other relational expressions to use
+    // fields.
+    for (final CorrelationId correlation : rel.getVariablesSet()) {
+      rel.accept(
+          new CorrelationReferenceFinder() {
+            protected RexNode handle(RexFieldAccess fieldAccess) {
+              final RexCorrelVariable v =
+                  (RexCorrelVariable) fieldAccess.getReferenceExpr();
+              if (v.id.equals(correlation)) {
+                fieldsUsedBuilder.set(fieldAccess.getField().getIndex());
+              }
+              return fieldAccess;
+            }
+          });
+    }
+
+    return dispatchTrimFields(input, fieldsUsedBuilder.build(), extraFields);
   }
 
   /**
@@ -234,8 +254,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     }
     relBuilder.push(trimResult.left)
         .project(exprList, nameList);
-    return new TrimResult(
-        relBuilder.build(),
+    return result(relBuilder.build(),
         Mappings.createIdentity(fieldList.size()));
   }
 
@@ -268,11 +287,44 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
       assert newFieldCount > 0 : "rel has no fields after trim: " + rel;
     }
     if (newRel.equals(rel)) {
-      return new TrimResult(rel, mapping);
+      return result(rel, mapping);
     }
     return trimResult;
   }
 
+  private TrimResult result(RelNode r, final Mapping mapping) {
+    map.put(r, mapping);
+    final RexBuilder rexBuilder = relBuilder.getRexBuilder();
+    final RelNode r0 = r;
+    for (final CorrelationId correlation : r.getVariablesSet()) {
+      r = r.accept(
+          new CorrelationReferenceFinder() {
+            protected RexNode handle(RexFieldAccess fieldAccess) {
+              final RexCorrelVariable v =
+                  (RexCorrelVariable) fieldAccess.getReferenceExpr();
+              if (v.id.equals(correlation)
+                  && v.getType().getFieldCount() == mapping.getSourceCount()) {
+                final int old = fieldAccess.getField().getIndex();
+                final int new_ = mapping.getTarget(old);
+                final RelDataTypeFactory.FieldInfoBuilder typeBuilder =
+                    relBuilder.getTypeFactory().builder();
+                for (int target : Util.range(mapping.getTargetCount())) {
+                  typeBuilder.add(
+                      v.getType().getFieldList().get(mapping.getSource(target)));
+                }
+                final RexNode newV =
+                    rexBuilder.makeCorrel(typeBuilder.build(), v.id);
+                if (old != new_) {
+                  return rexBuilder.makeFieldAccess(newV, new_);
+                }
+              }
+              return fieldAccess;
+            }
+          });
+    }
+    return new TrimResult(r, mapping);
+  }
+
   /**
    * Visit method, per {@link org.apache.calcite.util.ReflectiveVisitor}.
    *
@@ -295,10 +347,8 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // We don't know how to trim this kind of relational expression, so give
     // it back intact.
     Util.discard(fieldsUsed);
-    return new TrimResult(
-        rel,
-        Mappings.createIdentity(
-            rel.getRowType().getFieldCount()));
+    return result(rel,
+        Mappings.createIdentity(rel.getRowType().getFieldCount()));
   }
 
   /**
@@ -336,9 +386,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // there's nothing we can do.
     if (newInput == input
         && fieldsUsed.cardinality() == fieldCount) {
-      return new TrimResult(
-          project,
-          Mappings.createIdentity(fieldCount));
+      return result(project, Mappings.createIdentity(fieldCount));
     }
 
     // Some parts of the system can't handle rows with zero fields, so
@@ -371,7 +419,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
 
     relBuilder.push(newInput);
     relBuilder.project(newProjects, newRowType.getFieldNames());
-    return new TrimResult(relBuilder.build(), mapping);
+    return result(relBuilder.build(), mapping);
   }
 
   /** Creates a project with a dummy column, to protect the parts of the system
@@ -388,13 +436,13 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     if (input.getRowType().getFieldCount() == 1) {
       // Input already has one field (and may in fact be a dummy project we
       // created for the child). We can't do better.
-      return new TrimResult(input, mapping);
+      return result(input, mapping);
     }
     final RexLiteral expr =
         cluster.getRexBuilder().makeExactLiteral(BigDecimal.ZERO);
     relBuilder.push(input);
     relBuilder.project(ImmutableList.<RexNode>of(expr), ImmutableList.of("DUMMY"));
-    return new TrimResult(relBuilder.build(), mapping);
+    return result(relBuilder.build(), mapping);
   }
 
   /**
@@ -430,9 +478,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // there's nothing we can do.
     if (newInput == input
         && fieldsUsed.cardinality() == fieldCount) {
-      return new TrimResult(
-          filter,
-          Mappings.createIdentity(fieldCount));
+      return result(filter, Mappings.createIdentity(fieldCount));
     }
 
     // Build new project expressions, and populate the mapping.
@@ -448,7 +494,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // The result has the same mapping as the input gave us. Sometimes we
     // return fields that the consumer didn't ask for, because the filter
     // needs them for its condition.
-    return new TrimResult(relBuilder.build(), inputMapping);
+    return result(relBuilder.build(), inputMapping);
   }
 
   /**
@@ -484,9 +530,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     if (newInput == input
         && inputMapping.isIdentity()
         && fieldsUsed.cardinality() == fieldCount) {
-      return new TrimResult(
-          sort,
-          Mappings.createIdentity(fieldCount));
+      return result(sort, Mappings.createIdentity(fieldCount));
     }
 
     relBuilder.push(newInput);
@@ -501,7 +545,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // The result has the same mapping as the input gave us. Sometimes we
     // return fields that the consumer didn't ask for, because the filter
     // needs them for its condition.
-    return new TrimResult(relBuilder.build(), inputMapping);
+    return result(relBuilder.build(), inputMapping);
   }
 
   /**
@@ -609,7 +653,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
 
     if (changeCount == 0
         && mapping.isIdentity()) {
-      return new TrimResult(join, Mappings.createIdentity(fieldCount));
+      return result(join, Mappings.createIdentity(fieldCount));
     }
 
     // Build new join.
@@ -641,7 +685,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
       relBuilder.join(join.getJoinType(), newConditionExpr);
     }
 
-    return new TrimResult(relBuilder.build(), mapping);
+    return result(relBuilder.build(), mapping);
   }
 
   /**
@@ -701,9 +745,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
       for (RelNode input : setOp.getInputs()) {
         relBuilder.build();
       }
-      return new TrimResult(
-          setOp,
-          mapping);
+      return result(setOp, mapping);
     }
 
     switch (setOp.kind) {
@@ -720,7 +762,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     default:
       throw new AssertionError("unknown setOp " + setOp);
     }
-    return new TrimResult(relBuilder.build(), mapping);
+    return result(relBuilder.build(), mapping);
   }
 
   /**
@@ -779,8 +821,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // there's nothing to do.
     if (input == newInput
         && fieldsUsed.equals(ImmutableBitSet.range(rowType.getFieldCount()))) {
-      return new TrimResult(
-          aggregate,
+      return result(aggregate,
           Mappings.createIdentity(rowType.getFieldCount()));
     }
 
@@ -842,7 +883,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
         aggregate.indicator, newGroupSets);
     relBuilder.aggregate(groupKey, newAggCallList);
 
-    return new TrimResult(relBuilder.build(), mapping);
+    return result(relBuilder.build(), mapping);
   }
 
   /**
@@ -889,7 +930,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
 
     // Always project all fields.
     Mapping mapping = Mappings.createIdentity(fieldCount);
-    return new TrimResult(newModifier, mapping);
+    return result(newModifier, mapping);
   }
 
   /**
@@ -928,7 +969,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
 
     // Always project all fields.
     Mapping mapping = Mappings.createIdentity(fieldCount);
-    return new TrimResult(newTabFun, mapping);
+    return result(newTabFun, mapping);
   }
 
   /**
@@ -952,7 +993,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // If all fields are used, return unchanged.
     if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount))) {
       Mapping mapping = Mappings.createIdentity(fieldCount);
-      return new TrimResult(values, mapping);
+      return result(values, mapping);
     }
 
     final ImmutableList.Builder<ImmutableList<RexLiteral>> newTuples =
@@ -972,7 +1013,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     final LogicalValues newValues =
         LogicalValues.create(values.getCluster(), newRowType,
             newTuples.build());
-    return new TrimResult(newValues, mapping);
+    return result(newValues, mapping);
   }
 
   private Mapping createMapping(ImmutableBitSet fieldsUsed, int fieldCount) {
@@ -1024,7 +1065,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     }
 
     final Mapping mapping = createMapping(fieldsUsed, fieldCount);
-    return new TrimResult(newTableAccessRel, mapping);
+    return result(newTableAccessRel, mapping);
   }
 
   //~ Inner Classes ----------------------------------------------------------
@@ -1043,7 +1084,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
    *
    * <p>For example, consider the mapping for a relational expression that
    * has 4 output columns but only two are being used. The mapping
-   * {2 &rarr; 1, 3 &rarr; 0} would give the following behavior:</p>
+   * {2 &rarr; 1, 3 &rarr; 0} would give the following behavior:
    *
    * <ul>
    * <li>columnsUsed.getSourceCount() returns 4

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 31e8f88..bb37de1 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -2847,10 +2847,7 @@ public class SqlToRelConverter {
    * @return Whether to trim unused fields
    */
   public boolean isTrimUnusedFields() {
-    // To work around [CALCITE-842] "Decorrelator gets field offsets confused if
-    // fields have been trimmed", if expansion is disabled, trim fields after
-    // expansion and decorrelation.
-    return trimUnusedFields && expand;
+    return trimUnusedFields;
   }
 
   public void setExpand(boolean expand) {
@@ -3278,14 +3275,8 @@ public class SqlToRelConverter {
     } else {
       qualified = SqlQualified.create(null, 1, null, identifier);
     }
-    RexNode e = bb.lookupExp(qualified);
-    final CorrelationId correlationName;
-    if (e instanceof RexCorrelVariable) {
-      correlationName = ((RexCorrelVariable) e).id;
-    } else {
-      correlationName = null;
-    }
-
+    final RexNode e0 = bb.lookupExp(qualified);
+    RexNode e = e0;
     for (String name : qualified.suffixTranslated()) {
       final boolean caseSensitive = true; // name already fully-qualified
       e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
@@ -3295,10 +3286,11 @@ public class SqlToRelConverter {
       e = adjustInputRef(bb, (RexInputRef) e);
     }
 
-    if (null != correlationName) {
+    if (e0 instanceof RexCorrelVariable) {
       assert e instanceof RexFieldAccess;
       final RexNode prev =
-          bb.mapCorrelateToRex.put(correlationName, (RexFieldAccess) e);
+          bb.mapCorrelateToRex.put(((RexCorrelVariable) e0).id,
+              (RexFieldAccess) e);
       assert prev == null;
     }
     return e;

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java
index 3615003..51a4a70 100644
--- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java
@@ -103,20 +103,20 @@ public class JdbcAdapterTest {
             + "from scott.emp e inner join scott.dept d \n"
             + "on e.deptno = d.deptno")
         .explainContains("PLAN=JdbcToEnumerableConverter\n"
-            + "  JdbcProject(EMPNO=[$2], ENAME=[$3], DEPTNO=[$4], DNAME=[$1])\n"
-            + "    JdbcJoin(condition=[=($4, $0)], joinType=[inner])\n"
-            + "      JdbcProject(DEPTNO=[$0], DNAME=[$1])\n"
-            + "        JdbcTableScan(table=[[SCOTT, DEPT]])\n"
+            + "  JdbcProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$2], DNAME=[$4])\n"
+            + "    JdbcJoin(condition=[=($2, $3)], joinType=[inner])\n"
             + "      JdbcProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7])\n"
-            + "        JdbcTableScan(table=[[SCOTT, EMP]])")
+            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "      JdbcProject(DEPTNO=[$0], DNAME=[$1])\n"
+            + "        JdbcTableScan(table=[[SCOTT, DEPT]])")
         .runs()
         .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB)
-        .planHasSql("SELECT \"t0\".\"EMPNO\", \"t0\".\"ENAME\", "
-            + "\"t0\".\"DEPTNO\", \"t\".\"DNAME\"\n"
-            + "FROM (SELECT \"DEPTNO\", \"DNAME\"\n"
-            + "FROM \"SCOTT\".\"DEPT\") AS \"t\"\n"
-            + "INNER JOIN (SELECT \"EMPNO\", \"ENAME\", \"DEPTNO\"\n"
-            + "FROM \"SCOTT\".\"EMP\") AS \"t0\" "
+        .planHasSql("SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\", "
+            + "\"t\".\"DEPTNO\", \"t0\".\"DNAME\"\n"
+            + "FROM (SELECT \"EMPNO\", \"ENAME\", \"DEPTNO\"\n"
+            + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n"
+            + "INNER JOIN (SELECT \"DEPTNO\", \"DNAME\"\n"
+            + "FROM \"SCOTT\".\"DEPT\") AS \"t0\" "
             + "ON \"t\".\"DEPTNO\" = \"t0\".\"DEPTNO\"");
   }
 
@@ -129,20 +129,17 @@ public class JdbcAdapterTest {
             + "from scott.emp e inner join scott.salgrade s \n"
             + "on e.sal > s.losal and e.sal < s.hisal")
         .explainContains("PLAN=JdbcToEnumerableConverter\n"
-            + "  JdbcProject(EMPNO=[$3], ENAME=[$4], GRADE=[$0])\n"
-            + "    JdbcJoin(condition=[AND(>($5, $1), <($5, $2))], joinType=[inner])\n"
-            + "      JdbcTableScan(table=[[SCOTT, SALGRADE]])\n"
+            + "  JdbcProject(EMPNO=[$0], ENAME=[$1], GRADE=[$3])\n"
+            + "    JdbcJoin(condition=[AND(>($2, $4), <($2, $5))], joinType=[inner])\n"
             + "      JdbcProject(EMPNO=[$0], ENAME=[$1], SAL=[$5])\n"
-            + "        JdbcTableScan(table=[[SCOTT, EMP]])")
+            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "      JdbcTableScan(table=[[SCOTT, SALGRADE]])")
         .runs()
         .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB)
         .planHasSql("SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\", "
             + "\"SALGRADE\".\"GRADE\"\n"
-            + "FROM \"SCOTT\".\"SALGRADE\"\n"
-            + "INNER JOIN (SELECT \"EMPNO\", \"ENAME\", \"SAL\"\n"
-            + "FROM \"SCOTT\".\"EMP\") AS \"t\" "
-            + "ON \"SALGRADE\".\"LOSAL\" < \"t\".\"SAL\" "
-            + "AND \"SALGRADE\".\"HISAL\" > \"t\".\"SAL\"");
+            + "FROM (SELECT \"EMPNO\", \"ENAME\", \"SAL\"\nFROM \"SCOTT\".\"EMP\") AS \"t\"\n"
+            + "INNER JOIN \"SCOTT\".\"SALGRADE\" ON \"t\".\"SAL\" > \"SALGRADE\".\"LOSAL\"
AND \"t\".\"SAL\" < \"SALGRADE\".\"HISAL\"");
   }
 
   @Test public void testNonEquiJoinReverseConditionPlan() {
@@ -151,20 +148,18 @@ public class JdbcAdapterTest {
             + "from scott.emp e inner join scott.salgrade s \n"
             + "on s.losal <= e.sal and s.hisal >= e.sal")
         .explainContains("PLAN=JdbcToEnumerableConverter\n"
-            + "  JdbcProject(EMPNO=[$3], ENAME=[$4], GRADE=[$0])\n"
-            + "    JdbcJoin(condition=[AND(<=($1, $5), >=($2, $5))], joinType=[inner])\n"
-            + "      JdbcTableScan(table=[[SCOTT, SALGRADE]])\n"
+            + "  JdbcProject(EMPNO=[$0], ENAME=[$1], GRADE=[$3])\n"
+            + "    JdbcJoin(condition=[AND(<=($4, $2), >=($5, $2))], joinType=[inner])\n"
             + "      JdbcProject(EMPNO=[$0], ENAME=[$1], SAL=[$5])\n"
-            + "        JdbcTableScan(table=[[SCOTT, EMP]])")
+            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "      JdbcTableScan(table=[[SCOTT, SALGRADE]])")
         .runs()
         .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB)
         .planHasSql("SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\", "
             + "\"SALGRADE\".\"GRADE\"\n"
-            + "FROM \"SCOTT\".\"SALGRADE\"\n"
-            + "INNER JOIN (SELECT \"EMPNO\", \"ENAME\", \"SAL\"\n"
-            + "FROM \"SCOTT\".\"EMP\") AS \"t\" "
-            + "ON \"SALGRADE\".\"LOSAL\" <= \"t\".\"SAL\" "
-            + "AND \"SALGRADE\".\"HISAL\" >= \"t\".\"SAL\"");
+            + "FROM (SELECT \"EMPNO\", \"ENAME\", \"SAL\"\n"
+            + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n"
+            + "INNER JOIN \"SCOTT\".\"SALGRADE\" ON \"t\".\"SAL\" >= \"SALGRADE\".\"LOSAL\"
AND \"t\".\"SAL\" <= \"SALGRADE\".\"HISAL\"");
   }
 
   @Test public void testMixedJoinPlan() {
@@ -173,19 +168,20 @@ public class JdbcAdapterTest {
             + "from scott.emp e inner join scott.emp m on  \n"
             + "e.mgr = m.empno and e.sal > m.sal")
         .explainContains("PLAN=JdbcToEnumerableConverter\n"
-            + "  JdbcProject(EMPNO=[$2], ENAME=[$3], EMPNO0=[$2], ENAME0=[$3])\n"
-            + "    JdbcJoin(condition=[AND(=($4, $0), >($5, $1))], joinType=[inner])\n"
-            + "      JdbcProject(EMPNO=[$0], SAL=[$5])\n"
-            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "  JdbcProject(EMPNO=[$0], ENAME=[$1], EMPNO0=[$0], ENAME0=[$1])\n"
+            + "    JdbcJoin(condition=[AND(=($2, $4), >($3, $5))], joinType=[inner])\n"
             + "      JdbcProject(EMPNO=[$0], ENAME=[$1], MGR=[$3], SAL=[$5])\n"
+            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "      JdbcProject(EMPNO=[$0], SAL=[$5])\n"
             + "        JdbcTableScan(table=[[SCOTT, EMP]])")
         .runs()
         .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB)
-        .planHasSql("SELECT \"t0\".\"EMPNO\", \"t0\".\"ENAME\", "
-            + "\"t0\".\"EMPNO\" AS \"EMPNO0\", \"t0\".\"ENAME\" AS \"ENAME0\"\n"
-            + "FROM (SELECT \"EMPNO\", \"SAL\"\nFROM \"SCOTT\".\"EMP\") AS \"t\"\n"
-            + "INNER JOIN (SELECT \"EMPNO\", \"ENAME\", \"MGR\", \"SAL\"\n"
-            + "FROM \"SCOTT\".\"EMP\") AS \"t0\" ON \"t\".\"EMPNO\" = \"t0\".\"MGR\" AND
\"t\".\"SAL\" < \"t0\".\"SAL\"");
+        .planHasSql("SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\", "
+            + "\"t\".\"EMPNO\" AS \"EMPNO0\", \"t\".\"ENAME\" AS \"ENAME0\"\n"
+            + "FROM (SELECT \"EMPNO\", \"ENAME\", \"MGR\", \"SAL\"\n"
+            + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n"
+            + "INNER JOIN (SELECT \"EMPNO\", \"SAL\"\n"
+            + "FROM \"SCOTT\".\"EMP\") AS \"t0\" ON \"t\".\"MGR\" = \"t0\".\"EMPNO\" AND
\"t\".\"SAL\" > \"t0\".\"SAL\"");
   }
 
   @Test public void testMixedJoinWithOrPlan() {
@@ -194,23 +190,20 @@ public class JdbcAdapterTest {
             + "from scott.emp e inner join scott.emp m on  \n"
             + "e.mgr = m.empno and (e.sal > m.sal or m.hiredate > e.hiredate)")
         .explainContains("PLAN=JdbcToEnumerableConverter\n"
-            + "  JdbcProject(EMPNO=[$3], ENAME=[$4], EMPNO0=[$3], ENAME0=[$4])\n"
-            + "    JdbcJoin(condition=[AND(=($5, $0), OR(>($7, $2), >($1, $6)))], joinType=[inner])\n"
-            + "      JdbcProject(EMPNO=[$0], HIREDATE=[$4], SAL=[$5])\n"
-            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "  JdbcProject(EMPNO=[$0], ENAME=[$1], EMPNO0=[$0], ENAME0=[$1])\n"
+            + "    JdbcJoin(condition=[AND(=($2, $5), OR(>($4, $7), >($6, $3)))], joinType=[inner])\n"
             + "      JdbcProject(EMPNO=[$0], ENAME=[$1], MGR=[$3], HIREDATE=[$4], SAL=[$5])\n"
+            + "        JdbcTableScan(table=[[SCOTT, EMP]])\n"
+            + "      JdbcProject(EMPNO=[$0], HIREDATE=[$4], SAL=[$5])\n"
             + "        JdbcTableScan(table=[[SCOTT, EMP]])")
         .runs()
         .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB)
-        .planHasSql("SELECT \"t0\".\"EMPNO\", \"t0\".\"ENAME\", "
-            + "\"t0\".\"EMPNO\" AS \"EMPNO0\", \"t0\".\"ENAME\" AS \"ENAME0\"\n"
-            + "FROM (SELECT \"EMPNO\", \"HIREDATE\", \"SAL\"\n"
+        .planHasSql("SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\", "
+            + "\"t\".\"EMPNO\" AS \"EMPNO0\", \"t\".\"ENAME\" AS \"ENAME0\"\n"
+            + "FROM (SELECT \"EMPNO\", \"ENAME\", \"MGR\", \"HIREDATE\", \"SAL\"\n"
             + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n"
-            + "INNER JOIN (SELECT \"EMPNO\", \"ENAME\", \"MGR\", \"HIREDATE\", \"SAL\"\n"
-            + "FROM \"SCOTT\".\"EMP\") AS \"t0\" "
-            + "ON \"t\".\"EMPNO\" = \"t0\".\"MGR\" "
-            + "AND (\"t\".\"SAL\" < \"t0\".\"SAL\" "
-            + "OR \"t\".\"HIREDATE\" > \"t0\".\"HIREDATE\")");
+            + "INNER JOIN (SELECT \"EMPNO\", \"HIREDATE\", \"SAL\"\n"
+            + "FROM \"SCOTT\".\"EMP\") AS \"t0\" ON \"t\".\"MGR\" = \"t0\".\"EMPNO\" AND
(\"t\".\"SAL\" > \"t0\".\"SAL\" OR \"t\".\"HIREDATE\" < \"t0\".\"HIREDATE\")");
   }
 
   @Test public void testJoin3TablesPlan() {

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/java/org/apache/calcite/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
index ac9c56b..41a462a 100644
--- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java
+++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
@@ -3334,10 +3334,12 @@ public class JdbcTest {
       CalciteAssert.hr()
           .query("select count(*) c from \"hr\".\"emps\", \"hr\".\"depts\"")
           .convertContains("LogicalAggregate(group=[{}], C=[COUNT()])\n"
-              + "  LogicalProject($f0=[0])\n"
+              + "  LogicalProject(DUMMY=[0])\n"
               + "    LogicalJoin(condition=[true], joinType=[inner])\n"
-              + "      EnumerableTableScan(table=[[hr, emps]])\n"
-              + "      EnumerableTableScan(table=[[hr, depts]])");
+              + "      LogicalProject(DUMMY=[0])\n"
+              + "        EnumerableTableScan(table=[[hr, emps]])\n"
+              + "      LogicalProject(DUMMY=[0])\n"
+              + "        EnumerableTableScan(table=[[hr, depts]])");
     }
   }
 

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/java/org/apache/calcite/test/LatticeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
index 30ff1e1..be87b1c 100644
--- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java
+++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
@@ -233,7 +233,7 @@ public class LatticeTest {
           .convertMatches(
               CalciteAssert.checkRel(""
                   + "LogicalAggregate(group=[{}], EXPR$0=[COUNT()])\n"
-                  + "  LogicalProject($f0=[0])\n"
+                  + "  LogicalProject(DUMMY=[0])\n"
                   + "    StarTableScan(table=[[adhoc, star]])\n",
                   counter));
     } catch (RuntimeException e) {

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java b/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
index 9cb4240..7e818af 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
@@ -518,7 +518,7 @@ public abstract class SqlToRelTestBase {
       }
       if (enableTrim) {
         converter.setTrimUnusedFields(true);
-        root = root.withRel(converter.trimUnusedFields(false, root.rel));
+        root = root.withRel(converter.trimUnusedFields(true, root.rel));
       }
       return root;
     }

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
index 101a6ae..f1a7c67 100644
--- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
+++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java
@@ -52,9 +52,10 @@ public class EnumerableCorrelateTest {
         .query(
             "select empid, name from emps e where exists (select 1 from depts d where d.deptno=e.deptno)")
         .explainContains(""
-            + "EnumerableCalc(expr#0..5=[{inputs}], empid=[$t0], name=[$t2])\n"
+            + "EnumerableCalc(expr#0..3=[{inputs}], empid=[$t0], name=[$t2])\n"
             + "  EnumerableCorrelate(correlation=[$cor0], joinType=[INNER], requiredColumns=[{1}])\n"
-            + "    EnumerableTableScan(table=[[s, emps]])\n"
+            + "    EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}])\n"
+            + "      EnumerableTableScan(table=[[s, emps]])\n"
             + "    EnumerableAggregate(group=[{0}])\n"
             + "      EnumerableCalc(expr#0..3=[{inputs}], expr#4=[true], expr#5=[$cor0],
expr#6=[$t5.deptno], expr#7=[=($t0, $t6)], i=[$t4], $condition=[$t7])\n"
             + "        EnumerableTableScan(table=[[s, depts]])")

http://git-wip-us.apache.org/repos/asf/calcite/blob/4b519b98/core/src/test/resources/sql/subquery.iq
----------------------------------------------------------------------
diff --git a/core/src/test/resources/sql/subquery.iq b/core/src/test/resources/sql/subquery.iq
index b9964b6..cfeb6e5 100644
--- a/core/src/test/resources/sql/subquery.iq
+++ b/core/src/test/resources/sql/subquery.iq
@@ -48,8 +48,8 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t1, $t5)], expr#7=[f
             EnumerableValues(tuples=[[{ 0 }]])
           EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[=($t1, $t1)], expr#3=[null],
expr#4=[3], expr#5=[CASE($t2, $t3, $t4)], EXPR$0=[$t5])
             EnumerableValues(tuples=[[{ 0 }]])
-    EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}])
-      EnumerableAggregate(group=[{0}])
+    EnumerableAggregate(group=[{0, 1}])
+      EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}])
         EnumerableUnion(all=[true])
           EnumerableCalc(expr#0=[{inputs}], expr#1=[1], EXPR$0=[$t1])
             EnumerableValues(tuples=[[{ 0 }]])
@@ -274,6 +274,7 @@ GROUP BY emp.deptno;
 
 !ok
 
+!if (fixed.calcite1045) {
 # Correlated IN sub-query in WHERE clause of JOIN
 select empno from "scott".emp as e
 join "scott".dept as d using (deptno)
@@ -306,6 +307,7 @@ EnumerableCalc(expr#0..5=[{inputs}], EMPNO=[$t0])
     EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0])
       EnumerableTableScan(table=[[scott, DEPT]])
 !plan
+!}
 
 !if (fixed.calcite1045) {
 # Correlated NOT IN sub-query in WHERE clause of JOIN


Mime
View raw message