kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [1/3] kylin git commit: KYLIN-2277 return correct columns for select * [Forced Update!]
Date Fri, 16 Dec 2016 04:30:52 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 6317d2dce -> 8705eea69 (forced update)


KYLIN-2277 return correct columns for select *


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed497fe7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed497fe7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed497fe7

Branch: refs/heads/yang22-hbase1.x
Commit: ed497fe73bddffaf9373599c216400b323242833
Parents: 75ded9b
Author: Yang Li <liyang@apache.org>
Authored: Tue Dec 13 23:45:55 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Wed Dec 14 21:29:50 2016 +0800

----------------------------------------------------------------------
 .../calcite/sql2rel/SqlToRelConverter.java      | 62 +++++++++++++++++++-
 .../kylin/metadata/model/FunctionDesc.java      |  4 +-
 .../apache/kylin/query/ITKylinQueryTest.java    |  6 ++
 .../org/apache/kylin/query/KylinTestBase.java   | 21 +++++++
 .../resources/query/sql_timeout/query01.sql     |  2 +-
 5 files changed, 91 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed497fe7/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index d223cdf..cf36f61 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -67,6 +67,7 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -194,6 +195,7 @@ import static org.apache.calcite.util.Static.RESOURCE;
  * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
  * - isTrimUnusedFields(), override to false
  * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
+ * - convertQuery(), call hackSelectStar() at the end
  */
 
 /**
@@ -529,6 +531,8 @@ public class SqlToRelConverter {
      *                        the query will be part of a view.
      */
     public RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean
top) {
+        SqlNode origQuery = query; /* OVERRIDE POINT */
+        
         if (needsValidation) {
             query = validator.validate(query);
         }
@@ -553,7 +557,63 @@ public class SqlToRelConverter {
         }
 
         final RelDataType validatedRowType = validator.getValidatedNodeType(query);
-        return RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation);
+        return hackSelectStar(origQuery, RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation));
+    }
+
+    /* OVERRIDE POINT */
+    private RelRoot hackSelectStar(SqlNode query, RelRoot root) {
+        /*
+         * Rel tree is like:
+         * 
+         *   LogicalSort (optional)
+         *    |- LogicalProject
+         *        |- OLAPTableScan
+         */
+        LogicalProject rootPrj = null;
+        LogicalSort rootSort = null;
+        if (root.rel instanceof LogicalProject) {
+            rootPrj = (LogicalProject) root.rel;
+        } else if (root.rel instanceof LogicalSort && root.rel.getInput(0) instanceof
LogicalProject) {
+            rootPrj = (LogicalProject) root.rel.getInput(0);
+            rootSort = (LogicalSort) root.rel;
+        } else {
+            return root;
+        }
+        
+        if (!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+            return root;
+
+        RelNode scan = rootPrj.getInput();
+        if (rootPrj.getRowType().getFieldCount() < scan.getRowType().getFieldCount())
+            return root;
+        
+        RelDataType inType = rootPrj.getRowType();
+        List<String> inFields = inType.getFieldNames();
+        List<RexNode> projExp = new ArrayList<>();
+        List<Pair<Integer, String>> projFields = new ArrayList<>();
+        FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder();
+        FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder();
+        for (int i = 0; i < inFields.size(); i++) {
+            if (!inFields.get(i).startsWith("_KY_")) {
+                projExp.add(rootPrj.getProjects().get(i));
+                projFields.add(Pair.of(projFields.size(), inFields.get(i)));
+                projTypeBuilder.add(inType.getFieldList().get(i));
+                validTypeBuilder.add(root.validatedRowType.getFieldList().get(i));
+            }
+        }
+
+        RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder);
+        rootPrj = LogicalProject.create(scan, projExp, projRowType);
+        if (rootSort != null) {
+            rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation,
rootSort.offset, rootSort.fetch);
+        }
+        
+        RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder);
+        root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind,
projFields, root.collation);
+        
+        validator.setValidatedNodeType(query, validRowType);
+        
+        return root;
     }
 
     private static boolean isStream(SqlNode query) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed497fe7/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 7b41552..b2a4825 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -132,9 +132,9 @@ public class FunctionDesc {
         if (isSum()) {
             return getParameter().getValue();
         } else if (isCount()) {
-            return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all
the same
+            return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col)
are all the same
         } else {
-            return getFullExpression().replaceAll("[(), ]", "_");
+            return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_");
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed497fe7/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 90324b5..0379bd5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -390,4 +390,10 @@ public class ITKylinQueryTest extends KylinTestBase {
         // compare the result
         Assert.assertEquals(expectVersion, queriedVersion);
     }
+    
+    @Test
+    public void testSelectStarColumnCount() throws Exception {
+        execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
+        execAndCompColumnCount("select * from test_kylin_fact", 9);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed497fe7/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 52461c4..114a28d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -68,6 +68,7 @@ import org.dbunit.ext.h2.H2Connection;
 import org.dbunit.ext.h2.H2DataTypeFactory;
 import org.junit.Assert;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.io.Files;
 
 /**
@@ -423,6 +424,26 @@ public class KylinTestBase {
         }
     }
 
+    protected void execAndCompColumnCount(String input, int expectedColumnCount) throws Exception
{
+        printInfo("---------- test column count: " + input);
+        Set<String> sqlSet = ImmutableSet.of(input);
+        
+        for (String sql : sqlSet) {
+            // execute Kylin
+            printInfo("Query Result from Kylin - " + sql);
+            IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
+            ITable kylinTable = executeQuery(kylinConn, sql, sql, false);
+            
+            try {
+                // compare the result
+                Assert.assertEquals(expectedColumnCount, kylinTable.getTableMetaData().getColumns().length);
+            } catch (Throwable t) {
+                printInfo("execAndCompColumnCount failed on: " + sql);
+                throw t;
+            }
+        }
+    }
+    
     protected void execLimitAndValidate(String queryFolder) throws Exception {
         printInfo("---------- test folder: " + new File(queryFolder).getAbsolutePath());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed497fe7/kylin-it/src/test/resources/query/sql_timeout/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
index 3b9a837..eaff396 100644
--- a/kylin-it/src/test/resources/query/sql_timeout/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
@@ -16,4 +16,4 @@
 -- limitations under the License.
 --
 
-select * from test_kylin_fact limit 1200
+select seller_id,lstg_format_name from test_kylin_fact limit 1200


Mime
View raw message