kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [5/5] kylin git commit: KYLIN-1875 Add JoinsTree, refactor ModelChooser
Date Mon, 28 Nov 2016 23:27:04 GMT
KYLIN-1875 Add JoinsTree, refactor ModelChooser


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bda8787d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bda8787d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bda8787d

Branch: refs/heads/master
Commit: bda8787d1a5c9bbe7d94798091763e1004a1eb61
Parents: a1a2a4a
Author: Yang Li <liyang@apache.org>
Authored: Mon Nov 28 07:49:11 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Tue Nov 29 07:25:29 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/dict/DictionaryManager.java    |   6 +-
 .../kylin/metadata/model/DataModelDesc.java     |  30 +-
 .../apache/kylin/metadata/model/JoinDesc.java   |  11 +
 .../apache/kylin/metadata/model/JoinsTree.java  | 130 ++++++
 .../apache/kylin/metadata/model/TblColRef.java  |   4 +
 .../kylin/metadata/MetadataManagerTest.java     |  11 +-
 .../kylin/metadata/model/JoinsTreeTest.java     |  74 ++++
 .../test_kylin_snowflake_sales_cube.json        | 268 ++++++++++++
 .../test_kylin_snowflake_model_desc.json        | 161 --------
 .../test_kylin_snowflake_sales_model.json       |  87 ++++
 .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json |  28 ++
 .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json  | 408 +++++++++++++++++++
 .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json      | 152 +++++++
 .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json |  24 ++
 .../localmeta/table/SNOWTEST.KYLIN_SALES.json   |  56 +++
 .../apache/kylin/query/relnode/OLAPContext.java |   2 +
 .../kylin/query/relnode/OLAPTableScan.java      |   7 +-
 .../kylin/query/routing/ModelChooser.java       |  55 +--
 18 files changed, 1293 insertions(+), 221 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 820299c..6178234 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -339,7 +339,7 @@ public class DictionaryManager {
         
         // find a lookup table that the col joins as FK
         for (TableRef lookup : model.getLookupTables()) {
-            JoinDesc lookupJoin = model.getPKSideJoinMap().get(lookup);
+            JoinDesc lookupJoin = model.getJoinByPKSide(lookup);
             int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), col);
             if (find < 0)
                 continue;
@@ -357,8 +357,8 @@ public class DictionaryManager {
             if (join.isInnerJoin() == false)
                 return false;
             
-            TableRef table = join.getForeignKeyColumns()[0].getTableRef();
-            join = model.getPKSideJoinMap().get(table);
+            TableRef table = join.getFKSide();
+            join = model.getJoinByPKSide(table);
         }
         return true;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index ff92def..d917571 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -97,8 +97,7 @@ public class DataModelDesc extends RootPersistentEntity {
     private Set<TableRef> allTableRefs = Sets.newLinkedHashSet();
     private Map<String, TableRef> aliasMap = Maps.newHashMap(); // alias => TableRef, a table has exactly one alias
     private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // name => TableRef, a table maybe referenced by multiple names
-    private Map<TableRef, JoinDesc> pkSideJoinMap = Maps.newHashMap(); // table (PK side) => JoinTable
-    private Map<String, List<JoinDesc>> fkSideJoinMap = Maps.newHashMap(); // table (FK side) => JoinDesc
+    private JoinsTree joinsTree;
 
     /**
      * Error messages during resolving json metadata
@@ -151,12 +150,12 @@ public class DataModelDesc extends RootPersistentEntity {
         return joinTables;
     }
 
-    public Map<TableRef, JoinDesc> getPKSideJoinMap() {
-        return pkSideJoinMap;
+    public JoinDesc getJoinByPKSide(TableRef table) {
+        return joinsTree.getJoinByPKSide(table);
     }
-
-    public Map<String, List<JoinDesc>> getFKSideJoinMap() {
-        return fkSideJoinMap;
+    
+    public JoinsTree getJoinsTree() {
+        return joinsTree;
     }
 
     @Deprecated
@@ -282,6 +281,7 @@ public class DataModelDesc extends RootPersistentEntity {
         initJoinTablesForUpgrade();
         initTableAlias(tables);
         initJoinColumns();
+        initJoinsTree();
         ModelDimensionDesc.capicalizeStrings(dimensions);
         initPartitionDesc();
     }
@@ -360,8 +360,6 @@ public class DataModelDesc extends RootPersistentEntity {
     }
 
     private void initJoinColumns() {
-        pkSideJoinMap.clear();
-        fkSideJoinMap.clear();
 
         for (JoinTableDesc joinTable : joinTables) {
             TableRef dimTable = joinTable.getTableRef();
@@ -413,15 +411,15 @@ public class DataModelDesc extends RootPersistentEntity {
                     logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
                 }
             }
+        }
+    }
 
-            // pk/fk side join maps
-            pkSideJoinMap.put(dimTable, join);
-            List<JoinDesc> list = fkSideJoinMap.get(fkTable.getTableIdentity());
-            if (list == null) {
-                fkSideJoinMap.put(fkTable.getTableIdentity(), list = Lists.newArrayListWithCapacity(4));
-            }
-            list.add(join);
+    private void initJoinsTree() {
+        List<JoinDesc> joins = new ArrayList<>();
+        for (JoinTableDesc joinTable : joinTables) {
+            joins.add(joinTable.getJoin());
         }
+        joinsTree = new JoinsTree(rootFactTableRef, joins);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index d3c0745..6489244 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -110,6 +110,14 @@ public class JoinDesc {
             Preconditions.checkState(tableRef == cols[i].getTableRef());
     }
 
+    public TableRef getPKSide() {
+        return primaryKeyColumns[0].getTableRef();
+    }
+    
+    public TableRef getFKSide() {
+        return foreignKeyColumns[0].getTableRef();
+    }
+
     public void sortByFK() {
         Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length == primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length);
         boolean cont = true;
@@ -174,6 +182,9 @@ public class JoinDesc {
 
     // equals() without alias
     public boolean matches(JoinDesc other) {
+        if (other == null)
+            return false;
+        
         if (!this.type.equalsIgnoreCase(other.getType()))
             return false;
         

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
new file mode 100644
index 0000000..a0b267d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+public class JoinsTree {
+
+    final Map<String, Chain> tableChains = new LinkedHashMap<>();
+
+    public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
+        for (JoinDesc join : joins) {
+            for (TblColRef col : join.getForeignKeyColumns())
+                Preconditions.checkState(col.isQualified());
+            for (TblColRef col : join.getPrimaryKeyColumns())
+                Preconditions.checkState(col.isQualified());
+        }
+
+        tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null));
+
+        for (JoinDesc join : joins) {
+            TableRef pkSide = join.getPKSide();
+            Chain fkSide = tableChains.get(join.getFKSide().getAlias());
+            tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide));
+        }
+    }
+
+    public Map<String, String> matches(JoinsTree another) {
+        return matches(another, Collections.<String, String> emptyMap());
+    }
+
+    public Map<String, String> matches(JoinsTree another, Map<String, String> constraints) {
+        Map<String, String> matchUp = new HashMap<>();
+
+        for (Chain chain : tableChains.values()) {
+            if (matchInTree(chain, another, constraints, matchUp) == false)
+                return null;
+        }
+
+        return matchUp;
+    }
+
+    private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) {
+        String thisAlias = chain.table.getAlias();
+        if (matchUp.containsKey(thisAlias))
+            return true;
+
+        String constraint = constraints.get(thisAlias);
+        if (constraint != null) {
+            return matchChain(chain, another.tableChains.get(constraint), matchUp);
+        }
+
+        for (Chain anotherChain : another.tableChains.values()) {
+            if (matchChain(chain, anotherChain, matchUp)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean matchChain(Chain chain, Chain anotherChain, Map<String, String> matchUp) {
+        String thisAlias = chain.table.getAlias();
+        String anotherAlias = anotherChain.table.getAlias();
+
+        String curMatch = matchUp.get(thisAlias);
+        if (curMatch != null)
+            return curMatch.equals(anotherAlias);
+        if (curMatch == null && matchUp.values().contains(anotherAlias))
+            return false;
+
+        boolean matches = false;
+        if (chain.join == null) {
+            matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc());
+        } else {
+            matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp);
+        }
+
+        if (matches) {
+            matchUp.put(thisAlias, anotherAlias);
+        }
+        return matches;
+    }
+
+    public JoinDesc getJoinByPKSide(TableRef table) {
+        Chain chain = tableChains.get(table.getAlias());
+        if (chain == null)
+            return null;
+        else
+            return chain.join;
+    }
+
+    static class Chain {
+        TableRef table; // pk side
+        JoinDesc join;
+        Chain fkSide;
+
+        public Chain(TableRef table, JoinDesc join, Chain fkSide) {
+            this.table = table;
+            this.join = join;
+            this.fkSide = fkSide;
+            if (join != null) {
+                Preconditions.checkArgument(table == join.getPKSide());
+                Preconditions.checkArgument(fkSide.table == join.getFKSide());
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 2cfbafc..bf8d36b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -123,6 +123,10 @@ public class TblColRef implements Serializable {
         return table;
     }
     
+    public boolean isQualified() {
+        return table != null;
+    }
+    
     public String getTableAlias() {
         return table != null ? table.getAlias() : "UNKNOWN_ALIAS";
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index fcae8d4..204b4e0 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -76,17 +76,20 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
 
     @Test
     public void testSnowflakeDataModel() throws Exception {
-        DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_model_desc");
+        DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model");
         Assert.assertTrue(model.getDimensions().size() > 0);
 
         try {
-            model.findTable("TEST_KYLIN_COUNTRY");
+            model.findTable("KYLIN_COUNTRY");
             Assert.fail();
         } catch (IllegalArgumentException ex) {
             // excepted
         }
-        Assert.assertNotNull(model.findColumn("BUYER_COUNTRY"));
-        Assert.assertNotNull(model.findColumn("SELLER_COUNTRY"));
+        
+        Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+        Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+        Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+        Assert.assertNotNull(model.findColumn("BUYER_ID"));
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
new file mode 100644
index 0000000..f52ef45
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class JoinsTreeTest extends LocalFileMetadataTestCase {
+
+    @Before
+    public void setUp() throws Exception {
+        this.createTestMetadata();
+    }
+
+    @After
+    public void after() throws Exception {
+        this.cleanupTestMetadata();
+    }
+
+    @Test
+    public void testBasics() {
+        MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+        DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+        JoinsTree joinsTree = model.getJoinsTree();
+        
+        Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY");
+        assertTrue(chain.table == model.findTable("BUYER_COUNTRY"));
+        assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT"));
+        assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES"));
+        assertTrue(chain.fkSide.fkSide.join == null);
+        assertTrue(chain.fkSide.fkSide.fkSide == null);
+    }
+    
+    @Test
+    public void testMatch() {
+        MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+        DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+        JoinsTree joinsTree = model.getJoinsTree();
+
+        Map<String, String> matchUp = joinsTree.matches(joinsTree);
+        for (Entry<String, String> e : matchUp.entrySet()) {
+            assertTrue(e.getKey().equals(e.getValue()));
+        }
+        assertTrue(model.getAllTables().size() == matchUp.size());
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
new file mode 100644
index 0000000..ee0d68c
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
@@ -0,0 +1,268 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "test_kylin_snowflake_sales_cube",
+  "model_name" : "test_kylin_snowflake_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+    "name" : "TRANS_ID",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "TRANS_ID",
+    "derived" : null
+  }, {
+    "name" : "CAL_DT",
+    "table" : "SNOWTEST.KYLIN_CAL_DT",
+    "column" : "{FK}",
+    "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
+  }, {
+    "name" : "CATEGORY",
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "column" : "{FK}",
+    "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+    "name" : "META_CATEG_NAME",
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "column" : "META_CATEG_NAME",
+    "derived" : null
+  }, {
+    "name" : "CATEG_LVL2_NAME",
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "column" : "CATEG_LVL2_NAME",
+    "derived" : null
+  }, {
+    "name" : "CATEG_LVL3_NAME",
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "column" : "CATEG_LVL3_NAME",
+    "derived" : null
+  }, {
+    "name" : "LSTG_FORMAT_NAME",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "LSTG_FORMAT_NAME",
+    "derived" : null
+  }, {
+    "name" : "SELLER_ID",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "SELLER_ID",
+    "derived" : null
+  }, {
+    "name" : "BUYER_ID",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "BUYER_ID",
+    "derived" : null
+  }, {
+    "name" : "ACCOUNT_BUYER_LEVEL",
+    "table" : "BUYER_ACCOUNT",
+    "column" : "ACCOUNT_BUYER_LEVEL",
+    "derived" : null
+  }, {
+    "name" : "ACCOUNT_SELLER_LEVEL",
+    "table" : "SELLER_ACCOUNT",
+    "column" : "ACCOUNT_SELLER_LEVEL",
+    "derived" : null
+  }, {
+    "name" : "BUYER_COUNTRY",
+    "table" : "BUYER_ACCOUNT",
+    "column" : "ACCOUNT_COUNTRY",
+    "derived" : null
+  }, {
+    "name" : "SELLER_COUNTRY",
+    "table" : "SELLER_ACCOUNT",
+    "column" : "ACCOUNT_COUNTRY",
+    "derived" : null
+  }, {
+    "name" : "BUYER_COUNTRY_NAME",
+    "table" : "BUYER_COUNTRY",
+    "column" : "NAME",
+    "derived" : null
+  }, {
+    "name" : "SELLER_COUNTRY_NAME",
+    "table" : "SELLER_COUNTRY",
+    "column" : "NAME",
+    "derived" : null
+  }, {
+    "name" : "OPS_USER_ID",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "OPS_USER_ID",
+    "derived" : null
+  }, {
+    "name" : "OPS_REGION",
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "column" : "OPS_REGION",
+    "derived" : null
+  } ],
+  "measures" : [ {
+    "name" : "GMV_SUM",
+    "function" : {
+      "expression" : "SUM",
+      "parameter" : {
+        "type" : "column",
+        "value" : "PRICE",
+        "next_parameter" : null
+      },
+      "returntype" : "decimal(19,4)"
+    }
+  }, {
+    "name" : "GMV_MIN",
+    "function" : {
+      "expression" : "MIN",
+      "parameter" : {
+        "type" : "column",
+        "value" : "PRICE",
+        "next_parameter" : null
+      },
+      "returntype" : "decimal(19,4)"
+    }
+  }, {
+    "name" : "GMV_MAX",
+    "function" : {
+      "expression" : "MAX",
+      "parameter" : {
+        "type" : "column",
+        "value" : "PRICE",
+        "next_parameter" : null
+      },
+      "returntype" : "decimal(19,4)"
+    }
+  }, {
+    "name" : "TRANS_CNT",
+    "function" : {
+      "expression" : "COUNT",
+      "parameter" : {
+        "type" : "constant",
+        "value" : "1",
+        "next_parameter" : null
+      },
+      "returntype" : "bigint"
+    }
+  }, {
+    "name" : "SELLER_CNT_HLL",
+    "function" : {
+      "expression" : "COUNT_DISTINCT",
+      "parameter" : {
+        "type" : "column",
+        "value" : "SELLER_ID",
+        "next_parameter" : null
+      },
+      "returntype" : "hllc(10)"
+    }
+  }, {
+    "name" : "TOP_SELLER",
+    "function" : {
+      "expression" : "TOP_N",
+      "parameter" : {
+        "type" : "column",
+        "value" : "PRICE",
+        "next_parameter" : {
+          "type" : "column",
+          "value" : "SELLER_ID",
+          "next_parameter" : null
+        }
+      },
+      "returntype" : "topn(100)"
+    }
+  } ],
+  "rowkey" : {
+    "rowkey_columns" : [ {
+      "column" : "BUYER_ID",
+      "encoding" : "integer:4"
+    }, {
+      "column" : "SELLER_ID",
+      "encoding" : "integer:4"
+    }, {
+      "column" : "TRANS_ID",
+      "encoding" : "integer:4"
+    }, {
+      "column" : "PART_DT",
+      "encoding" : "date"
+    }, {
+      "column" : "LEAF_CATEG_ID",
+      "encoding" : "dict"
+    }, {
+      "column" : "META_CATEG_NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "CATEG_LVL2_NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "CATEG_LVL3_NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+      "encoding" : "dict"
+    }, {
+      "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+      "encoding" : "dict"
+    }, {
+      "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+      "encoding" : "dict"
+    }, {
+      "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+      "encoding" : "dict"
+    }, {
+      "column" : "BUYER_COUNTRY.NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "SELLER_COUNTRY.NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "LSTG_FORMAT_NAME",
+      "encoding" : "dict"
+    }, {
+      "column" : "LSTG_SITE_ID",
+      "encoding" : "dict"
+    }, {
+      "column" : "OPS_USER_ID",
+      "encoding" : "dict"
+    }, {
+      "column" : "OPS_REGION",
+      "encoding" : "dict"
+    } ]
+  },
+  "hbase_mapping" : {
+    "column_family" : [ {
+      "name" : "F1",
+      "columns" : [ {
+        "qualifier" : "M",
+        "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+      } ]
+    }, {
+      "name" : "F2",
+      "columns" : [ {
+        "qualifier" : "M",
+        "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
+      } ]
+    } ]
+  },
+  "aggregation_groups" : [ {
+    "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", 
+                   "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+    "select_rule" : {
+      "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+      "mandatory_dims" : [ "PART_DT" ],
+      "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
+                       [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
+    }
+  }, {
+    "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", 
+                   "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", 
+                   "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+    "select_rule" : {
+      "hierarchy_dims" : [ ],
+      "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", 
+                   "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", 
+                   "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+      "joint_dims" : [ ]
+    }
+  } ],
+  "notify_list" : null,
+  "status_need_notify" : [ ],
+  "partition_date_start" : 1325376000000,
+  "auto_merge_time_ranges" : null,
+  "retention_range" : 0,
+  "engine_type" : 2,
+  "storage_type" : 2,
+  "override_kylin_properties" : {
+    "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
+  }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
deleted file mode 100644
index b06a217..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
+++ /dev/null
@@ -1,161 +0,0 @@
-{
-  "uuid": "ac0f4ee2-1dcb-4b07-a38e-4c298563e0e3",
-  "name": "test_kylin_snowflake_model_desc",
-  "lookups": [
-    {
-      "table": "EDW.TEST_CAL_DT",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "CAL_DT"
-        ],
-        "foreign_key": [
-          "CAL_DT"
-        ]
-      }
-    },
-    {
-      "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "LEAF_CATEG_ID",
-          "SITE_ID"
-        ],
-        "foreign_key": [
-          "LEAF_CATEG_ID",
-          "LSTG_SITE_ID"
-        ]
-      }
-    },
-    {
-      "table": "EDW.TEST_SITES",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "SITE_ID"
-        ],
-        "foreign_key": [
-          "LSTG_SITE_ID"
-        ]
-      }
-    },
-    {
-      "table": "EDW.TEST_SELLER_TYPE_DIM",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "SELLER_TYPE_CD"
-        ],
-        "foreign_key": [
-          "SLR_SEGMENT_CD"
-        ]
-      }
-    },
-    {
-      "table": "DEFAULT.TEST_KYLIN_COUNTRY",
-      "alias": "BUYER_COUNTRY",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "COUNTRY"
-        ],
-        "foreign_key": [
-          "BUYER_COUNTRY"
-        ]
-      }
-    },
-    {
-      "table": "DEFAULT.TEST_KYLIN_COUNTRY",
-      "alias": "SELLER_COUNTRY",
-      "join": {
-        "type": "left",
-        "primary_key": [
-          "COUNTRY"
-        ],
-        "foreign_key": [
-          "SELLER_COUNTRY"
-        ]
-      }
-    }
-  ],
-  "dimensions": [
-    {
-      "table": "default.test_kylin_fact",
-      "columns": [
-        "lstg_format_name",
-        "LSTG_SITE_ID",
-        "SLR_SEGMENT_CD",
-        "TRANS_ID",
-        "CAL_DT",
-        "LEAF_CATEG_ID",
-        "SELLER_ID",
-        "BUYER_COUNTRY",
-        "SELLER_COUNTRY"
-      ]
-    },
-    {
-      "table": "default.test_category_groupings",
-      "columns": [
-        "leaf_categ_id",
-        "site_id",
-        "USER_DEFINED_FIELD1",
-        "USER_DEFINED_FIELD3",
-        "UPD_DATE",
-        "UPD_USER",
-        "meta_categ_name",
-        "categ_lvl2_name",
-        "categ_lvl3_name"
-      ]
-    },
-    {
-      "table": "edw.test_sites",
-      "columns": [
-        "site_id",
-        "site_name",
-        "cre_user"
-      ]
-    },
-    {
-      "table": "edw.test_seller_type_dim",
-      "columns": [
-        "seller_type_cd",
-        "seller_type_desc"
-      ]
-    },
-    {
-      "table": "edw.test_cal_dt",
-      "columns": [
-        "cal_dt",
-        "week_beg_dt"
-      ]
-    },
-    {
-      "table": "BUYER_COUNTRY",
-      "columns": [
-        "country",
-        "name"
-      ]
-    },
-    {
-      "table": "SELLER_COUNTRY",
-      "columns": [
-        "country",
-        "name"
-      ]
-    }
-  ],
-  "metrics": [
-    "PRICE",
-    "ITEM_COUNT",
-    "SELLER_ID"
-  ],
-  "last_modified": 1422435345352,
-  "fact_table": "DEFAULT.TEST_KYLIN_FACT",
-  "filter_condition": null,
-  "partition_desc": {
-    "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
-    "partition_date_start": 0,
-    "partition_type": "APPEND"
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
new file mode 100644
index 0000000..84c6979
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
@@ -0,0 +1,87 @@
+{
+  "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
+  "name" : "test_kylin_snowflake_sales_model",
+  "lookups" : [ {
+    "table" : "SNOWTEST.KYLIN_CAL_DT",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "CAL_DT" ],
+      "foreign_key" : [ "PART_DT" ]
+    }
+  }, {
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
+      "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
+    }
+  }, {
+    "table" : "SNOWTEST.KYLIN_ACCOUNT",
+    "alias" : "BUYER_ACCOUNT",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "ACCOUNT_ID" ],
+      "foreign_key" : [ "BUYER_ID" ]
+    }
+  }, {
+    "table" : "SNOWTEST.KYLIN_ACCOUNT",
+    "alias" : "SELLER_ACCOUNT",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "ACCOUNT_ID" ],
+      "foreign_key" : [ "SELLER_ID" ]
+    }
+  }, {
+    "table" : "SNOWTEST.KYLIN_COUNTRY",
+    "alias" : "BUYER_COUNTRY",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "COUNTRY" ],
+      "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
+    }
+  }, {
+    "table" : "SNOWTEST.KYLIN_COUNTRY",
+    "alias" : "SELLER_COUNTRY",
+    "join" : {
+      "type" : "inner",
+      "primary_key" : [ "COUNTRY" ],
+      "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
+    }
+  }],
+  "dimensions" : [ {
+    "table" : "SNOWTEST.KYLIN_SALES",
+    "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
+  }, {
+    "table" : "SNOWTEST.KYLIN_CAL_DT",
+    "columns" : ["CAL_DT", "WEEK_BEG_DT"]
+  }, {
+    "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+    "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
+  }, {
+    "table" : "BUYER_ACCOUNT",
+    "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+  }, {
+    "table" : "SELLER_ACCOUNT",
+    "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+  }, {
+    "table" : "BUYER_COUNTRY",
+    "columns" : [ "COUNTRY", "NAME" ]
+  }, {
+    "table" : "SELLER_COUNTRY",
+    "columns" : [ "COUNTRY", "NAME" ]
+  } ],
+  "metrics": [
+    "PRICE",
+    "ITEM_COUNT",
+    "SELLER_ID",
+    "LSTG_FORMAT_NAME"
+  ],
+  "last_modified" : 1422435345362,
+  "fact_table" : "SNOWTEST.KYLIN_SALES",
+  "filter_condition" : null,
+  "partition_desc" : {
+    "partition_date_column" : "SNOWTEST.KYLIN_SALES.PART_DT",
+    "partition_date_start" : 1325376000000,
+    "partition_type" : "APPEND"
+  }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
new file mode 100644
index 0000000..8d39cbc
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
@@ -0,0 +1,28 @@
+{
+  "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
+ 
+  "name" : "KYLIN_ACCOUNT",
+  "columns" : [ {
+    "id" : "1",
+    "name" : "ACCOUNT_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "2",
+    "name" : "ACCOUNT_BUYER_LEVEL",
+    "datatype" : "int"
+  }, {
+    "id" : "3",
+    "name" : "ACCOUNT_SELLER_LEVEL",
+    "datatype" : "int"
+  }, {
+    "id" : "4",
+    "name" : "ACCOUNT_COUNTRY",
+    "datatype" : "string"
+  }, {
+    "id" : "5",
+    "name" : "ACCOUNT_CONTACT",
+    "datatype" : "string"
+  } ],
+  "database" : "SNOWTEST",
+  "last_modified" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
new file mode 100644
index 0000000..655d4e7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -0,0 +1,408 @@
+{
+  "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ 
+  "name" : "KYLIN_CAL_DT",
+  "columns" : [ {
+    "id" : "1",
+    "name" : "CAL_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "2",
+    "name" : "YEAR_BEG_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "3",
+    "name" : "QTR_BEG_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "4",
+    "name" : "MONTH_BEG_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "5",
+    "name" : "WEEK_BEG_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "6",
+    "name" : "AGE_FOR_YEAR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "7",
+    "name" : "AGE_FOR_QTR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "8",
+    "name" : "AGE_FOR_MONTH_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "9",
+    "name" : "AGE_FOR_WEEK_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "10",
+    "name" : "AGE_FOR_DT_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "11",
+    "name" : "AGE_FOR_RTL_YEAR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "12",
+    "name" : "AGE_FOR_RTL_QTR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "13",
+    "name" : "AGE_FOR_RTL_MONTH_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "14",
+    "name" : "AGE_FOR_RTL_WEEK_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "15",
+    "name" : "AGE_FOR_CS_WEEK_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "16",
+    "name" : "DAY_OF_CAL_ID",
+    "datatype" : "int"
+  }, {
+    "id" : "17",
+    "name" : "DAY_OF_YEAR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "18",
+    "name" : "DAY_OF_QTR_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "19",
+    "name" : "DAY_OF_MONTH_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "20",
+    "name" : "DAY_OF_WEEK_ID",
+    "datatype" : "int"
+  }, {
+    "id" : "21",
+    "name" : "WEEK_OF_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "22",
+    "name" : "WEEK_OF_CAL_ID",
+    "datatype" : "int"
+  }, {
+    "id" : "23",
+    "name" : "MONTH_OF_QTR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "24",
+    "name" : "MONTH_OF_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "25",
+    "name" : "MONTH_OF_CAL_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "26",
+    "name" : "QTR_OF_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "27",
+    "name" : "QTR_OF_CAL_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "28",
+    "name" : "YEAR_OF_CAL_ID",
+    "datatype" : "smallint"
+  }, {
+    "id" : "29",
+    "name" : "YEAR_END_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "30",
+    "name" : "QTR_END_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "31",
+    "name" : "MONTH_END_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "32",
+    "name" : "WEEK_END_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "33",
+    "name" : "CAL_DT_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "34",
+    "name" : "CAL_DT_DESC",
+    "datatype" : "string"
+  }, {
+    "id" : "35",
+    "name" : "CAL_DT_SHORT_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "36",
+    "name" : "YTD_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "37",
+    "name" : "QTD_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "38",
+    "name" : "MTD_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "39",
+    "name" : "WTD_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "40",
+    "name" : "SEASON_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "41",
+    "name" : "DAY_IN_YEAR_COUNT",
+    "datatype" : "smallint"
+  }, {
+    "id" : "42",
+    "name" : "DAY_IN_QTR_COUNT",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "43",
+    "name" : "DAY_IN_MONTH_COUNT",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "44",
+    "name" : "DAY_IN_WEEK_COUNT",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "45",
+    "name" : "RTL_YEAR_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "46",
+    "name" : "RTL_QTR_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "47",
+    "name" : "RTL_MONTH_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "48",
+    "name" : "RTL_WEEK_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "49",
+    "name" : "CS_WEEK_BEG_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "50",
+    "name" : "CAL_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "51",
+    "name" : "DAY_OF_WEEK",
+    "datatype" : "string"
+  }, {
+    "id" : "52",
+    "name" : "MONTH_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "53",
+    "name" : "PRD_DESC",
+    "datatype" : "string"
+  }, {
+    "id" : "54",
+    "name" : "PRD_FLAG",
+    "datatype" : "string"
+  }, {
+    "id" : "55",
+    "name" : "PRD_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "56",
+    "name" : "PRD_IND",
+    "datatype" : "string"
+  }, {
+    "id" : "57",
+    "name" : "QTR_DESC",
+    "datatype" : "string"
+  }, {
+    "id" : "58",
+    "name" : "QTR_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "59",
+    "name" : "QTR_IND",
+    "datatype" : "string"
+  }, {
+    "id" : "60",
+    "name" : "RETAIL_WEEK",
+    "datatype" : "string"
+  }, {
+    "id" : "61",
+    "name" : "RETAIL_YEAR",
+    "datatype" : "string"
+  }, {
+    "id" : "62",
+    "name" : "RETAIL_START_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "63",
+    "name" : "RETAIL_WK_END_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "64",
+    "name" : "WEEK_IND",
+    "datatype" : "string"
+  }, {
+    "id" : "65",
+    "name" : "WEEK_NUM_DESC",
+    "datatype" : "string"
+  }, {
+    "id" : "66",
+    "name" : "WEEK_BEG_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "67",
+    "name" : "WEEK_END_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "68",
+    "name" : "WEEK_IN_YEAR_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "69",
+    "name" : "WEEK_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "70",
+    "name" : "WEEK_BEG_END_DESC_MDY",
+    "datatype" : "string"
+  }, {
+    "id" : "71",
+    "name" : "WEEK_BEG_END_DESC_MD",
+    "datatype" : "string"
+  }, {
+    "id" : "72",
+    "name" : "YEAR_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "73",
+    "name" : "YEAR_IND",
+    "datatype" : "string"
+  }, {
+    "id" : "74",
+    "name" : "CAL_DT_MNS_1YEAR_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "75",
+    "name" : "CAL_DT_MNS_2YEAR_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "76",
+    "name" : "CAL_DT_MNS_1QTR_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "77",
+    "name" : "CAL_DT_MNS_2QTR_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "78",
+    "name" : "CAL_DT_MNS_1MONTH_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "79",
+    "name" : "CAL_DT_MNS_2MONTH_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "80",
+    "name" : "CAL_DT_MNS_1WEEK_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "81",
+    "name" : "CAL_DT_MNS_2WEEK_DT",
+    "datatype" : "string"
+  }, {
+    "id" : "82",
+    "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "83",
+    "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "84",
+    "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "85",
+    "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "86",
+    "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "87",
+    "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "88",
+    "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "89",
+    "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "90",
+    "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "91",
+    "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "92",
+    "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "93",
+    "name" : "SEASON_OF_YEAR_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "94",
+    "name" : "YTM_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "95",
+    "name" : "YTQ_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "96",
+    "name" : "YTW_YN_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "97",
+    "name" : "KYLIN_CAL_DT_CRE_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "98",
+    "name" : "KYLIN_CAL_DT_CRE_USER",
+    "datatype" : "string"
+  }, {
+    "id" : "99",
+    "name" : "KYLIN_CAL_DT_UPD_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "100",
+    "name" : "KYLIN_CAL_DT_UPD_USER",
+    "datatype" : "string"
+  } ],
+  "database" : "SNOWTEST",
+  "last_modified" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
new file mode 100644
index 0000000..abee6d1
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
@@ -0,0 +1,152 @@
+{
+  "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+ 
+  "name" : "KYLIN_CATEGORY_GROUPINGS",
+  "columns" : [ {
+    "id" : "1",
+    "name" : "LEAF_CATEG_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "2",
+    "name" : "LEAF_CATEG_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "3",
+    "name" : "SITE_ID",
+    "datatype" : "int"
+  }, {
+    "id" : "4",
+    "name" : "CATEG_BUSN_MGR",
+    "datatype" : "string"
+  }, {
+    "id" : "5",
+    "name" : "CATEG_BUSN_UNIT",
+    "datatype" : "string"
+  }, {
+    "id" : "6",
+    "name" : "REGN_CATEG",
+    "datatype" : "string"
+  }, {
+    "id" : "7",
+    "name" : "USER_DEFINED_FIELD1",
+    "datatype" : "string"
+  }, {
+    "id" : "8",
+    "name" : "USER_DEFINED_FIELD3",
+    "datatype" : "string"
+  }, {
+    "id" : "9",
+    "name" : "KYLIN_GROUPINGS_CRE_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "10",
+    "name" : "KYLIN_GROUPINGS_UPD_DATE",
+    "datatype" : "string"
+  }, {
+    "id" : "11",
+    "name" : "KYLIN_GROUPINGS_CRE_USER",
+    "datatype" : "string"
+  }, {
+    "id" : "12",
+    "name" : "KYLIN_GROUPINGS_UPD_USER",
+    "datatype" : "string"
+  }, {
+    "id" : "13",
+    "name" : "META_CATEG_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "14",
+    "name" : "META_CATEG_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "15",
+    "name" : "CATEG_LVL2_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "16",
+    "name" : "CATEG_LVL3_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "17",
+    "name" : "CATEG_LVL4_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "18",
+    "name" : "CATEG_LVL5_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "19",
+    "name" : "CATEG_LVL6_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "20",
+    "name" : "CATEG_LVL7_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "21",
+    "name" : "CATEG_LVL2_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "22",
+    "name" : "CATEG_LVL3_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "23",
+    "name" : "CATEG_LVL4_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "24",
+    "name" : "CATEG_LVL5_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "25",
+    "name" : "CATEG_LVL6_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "26",
+    "name" : "CATEG_LVL7_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "27",
+    "name" : "CATEG_FLAGS",
+    "datatype" : "decimal"
+  }, {
+    "id" : "28",
+    "name" : "ADULT_CATEG_YN",
+    "datatype" : "string"
+  }, {
+    "id" : "29",
+    "name" : "DOMAIN_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "30",
+    "name" : "USER_DEFINED_FIELD5",
+    "datatype" : "string"
+  }, {
+    "id" : "31",
+    "name" : "VCS_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "32",
+    "name" : "GCS_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "33",
+    "name" : "MOVE_TO",
+    "datatype" : "decimal"
+  }, {
+    "id" : "34",
+    "name" : "SAP_CATEGORY_ID",
+    "datatype" : "decimal"
+  }, {
+    "id" : "35",
+    "name" : "SRC_ID",
+    "datatype" : "tinyint"
+  }, {
+    "id" : "36",
+    "name" : "BSNS_VRTCL_NAME",
+    "datatype" : "string"
+  } ],
+  "database" : "SNOWTEST",
+  "last_modified" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
new file mode 100644
index 0000000..64708c7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
@@ -0,0 +1,24 @@
+{
+  "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+ 
+  "name" : "KYLIN_COUNTRY",
+  "columns" : [ {
+    "id" : "1",
+    "name" : "COUNTRY",
+    "datatype" : "string"
+  }, {
+    "id" : "2",
+    "name" : "LATITUDE",
+    "datatype" : "double"
+  }, {
+    "id" : "3",
+    "name" : "LONGITUDE",
+    "datatype" : "double"
+  }, {
+    "id" : "4",
+    "name" : "NAME",
+    "datatype" : "string"
+  } ],
+  "database" : "SNOWTEST",
+  "last_modified" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
new file mode 100644
index 0000000..4aa82b5
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
@@ -0,0 +1,56 @@
+{
+  "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+ 
+  "name" : "KYLIN_SALES",
+  "columns" : [ {
+    "id" : "1",
+    "name" : "TRANS_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "2",
+    "name" : "PART_DT",
+    "datatype" : "date"
+  }, {
+    "id" : "3",
+    "name" : "LSTG_FORMAT_NAME",
+    "datatype" : "string"
+  }, {
+    "id" : "4",
+    "name" : "LEAF_CATEG_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "5",
+    "name" : "LSTG_SITE_ID",
+    "datatype" : "int"
+  }, {
+    "id" : "6",
+    "name" : "SLR_SEGMENT_CD",
+    "datatype" : "smallint"
+  }, {
+    "id" : "7",
+    "name" : "PRICE",
+    "datatype" : "decimal(19,4)"
+  }, {
+    "id" : "8",
+    "name" : "ITEM_COUNT",
+    "datatype" : "bigint"
+  }, {
+    "id" : "9",
+    "name" : "SELLER_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "10",
+    "name" : "BUYER_ID",
+    "datatype" : "bigint"
+  }, {
+    "id" : "11",
+    "name" : "OPS_USER_ID",
+    "datatype" : "string"
+  }, {
+    "id" : "12",
+    "name" : "OPS_REGION",
+    "datatype" : "string"
+  } ],
+  "database" : "SNOWTEST",
+  "last_modified" : 0
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 615d802..5fb2c5c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -33,6 +33,7 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.SQLDigest;
@@ -127,6 +128,7 @@ public class OLAPContext {
     public Set<TblColRef> filterColumns = new HashSet<>();
     public TupleFilter filter;
     public List<JoinDesc> joins = new LinkedList<>();
+    public JoinsTree joinsTree;
     private List<TblColRef> sortColumns;
     private List<SQLDigest.OrderEnum> sortOrders;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 3c2bf48..14758c9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -239,10 +239,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
         return new ColumnRowType(columns);
     }
     
+    public TableRef getTableRef() {
+        return columnRowType.getColumnByIndex(0).getTableRef();
+    }
+    
     @SuppressWarnings("deprecation")
     public TblColRef makeRewriteColumn(String name) {
-        TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef();
-        return tableRef.makeFakeColumn(name);
+        return getTableRef().makeFakeColumn(name);
     }
     
     public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index f1f5a48..ccb2900 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectManager;
@@ -38,7 +39,7 @@ import org.apache.kylin.query.relnode.OLAPContext;
 import org.apache.kylin.query.relnode.OLAPTableScan;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 
-import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
@@ -73,47 +74,31 @@ public class ModelChooser {
     private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) {
         Map<String, String> result = Maps.newHashMap();
 
-        // the greedy match is not perfect but works for the moment
-        Map<String, List<JoinDesc>> modelJoinsMap = model.getFKSideJoinMap();
         for (OLAPContext ctx : contexts) {
-            for (JoinDesc queryJoin : ctx.joins) {
-                String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
-                List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
-                if (modelJoins == null)
-                    return null;
-
-                JoinDesc matchJoin = null;
-                for (JoinDesc modelJoin : modelJoins) {
-                    if (modelJoin.matches(queryJoin)) {
-                        matchJoin = modelJoin;
-                        break;
-                    }
-                }
-                if (matchJoin == null)
-                    return null;
+            TableRef firstTable = ctx.firstTableScan.getTableRef();
 
-                matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
-                matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
-            }
-            
-            OLAPTableScan firstTable = ctx.firstTableScan;
-            String firstTableAlias = firstTable.getAlias();
-            if (result.containsKey(firstTableAlias) == false) {
-                TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
-                if (tableRef == null)
-                    return null;
-                matchesAdd(firstTableAlias, tableRef.getAlias(), result);
+            Map<String, String> matchUp = null;
+
+            if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) {
+                // one lookup table
+                String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias();
+                matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
+            } else {
+                // normal big joins
+                if (ctx.joinsTree == null) {
+                    ctx.joinsTree = new JoinsTree(firstTable, ctx.joins);
+                }
+                matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result);
             }
+
+            if (matchUp == null)
+                return null;
+
+            result.putAll(matchUp);
         }
-        
         return result;
     }
 
-    private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) {
-        String existingTarget = result.put(origAlias, targetAlias);
-        Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
-    }
-
     private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) {
         // the first context, which is the top most context, contains all columns from all contexts
         OLAPContext first = contexts.get(0);


Mime
View raw message