kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [12/50] [abbrv] kylin git commit: KYLIN-2575 Experimental feature: Computed Column
Date Fri, 19 May 2017 23:57:50 GMT
KYLIN-2575 Experimental feature: Computed Column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a812d5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a812d5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a812d5f

Branch: refs/heads/KYLIN-2624
Commit: 9a812d5fc917ddcdb736cd1c91fdfe9c933e603a
Parents: d6fdda2
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Mon May 8 15:49:57 2017 +0800
Committer: liyang-gmt8 <liyang@apache.org>
Committed: Mon May 8 16:01:48 2017 +0800

----------------------------------------------------------------------
 .../kylin/cube/CubeCapabilityChecker.java       |   11 +-
 .../java/org/apache/kylin/cube/JoinChecker.java |   60 -
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |    7 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |   12 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   23 +
 .../model/validation/rule/FunctionRuleTest.java |   11 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   |    4 +-
 .../apache/kylin/metadata/MetadataManager.java  |   31 +-
 .../apache/kylin/metadata/model/ColumnDesc.java |   41 +-
 .../metadata/model/ComputedColumnDesc.java      |   95 ++
 .../kylin/metadata/model/DataModelDesc.java     |   81 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |    2 +-
 .../kylin/metadata/model/PartitionDesc.java     |   10 +-
 .../apache/kylin/metadata/model/TableDesc.java  |   35 +-
 .../apache/kylin/metadata/model/TableRef.java   |   14 +-
 .../apache/kylin/metadata/model/TblColRef.java  |   36 +-
 .../kylin/metadata/project/ProjectL2Cache.java  |   13 +-
 .../kylin/metadata/model/DataModelDescTest.java |   15 +-
 .../localmeta/cube_desc/ci_inner_join_cube.json | 1016 ++++++++++-------
 .../localmeta/cube_desc/ci_left_join_cube.json  | 1036 ++++++++++--------
 .../model_desc/ci_inner_join_model.json         |   37 +-
 .../model_desc/ci_left_join_model.json          |   37 +-
 .../test_case_data/sandbox/kylin.properties     |    2 +-
 .../kylin/query/CompareQueryBySuffix.java       |   37 +
 .../kylin/query/ICompareQueryTranslator.java    |   29 +
 .../apache/kylin/query/ITKylinQueryTest.java    |    5 +
 .../org/apache/kylin/query/KylinTestBase.java   |   25 +-
 .../query/sql_computedcolumn/query01.sql        |   14 +
 .../sql_computedcolumn/query01.sql.compare      |   14 +
 .../query/sql_computedcolumn/query02.sql        |   21 +
 .../sql_computedcolumn/query02.sql.compare      |   21 +
 .../query/sql_verifyCount/query01.sql.expected  |    2 +-
 .../query/sql_verifyCount/query03.sql.expected  |    2 +-
 .../query/sql_verifyCount/query04.sql.expected  |    2 +-
 .../apache/kylin/query/schema/OLAPSchema.java   |    2 +-
 .../kylin/rest/controller/ModelController.java  |    2 +-
 .../kylin/rest/service/CacheServiceTest.java    |    1 -
 .../kylin/rest/service/ModelServiceTest.java    |   97 ++
 .../apache/kylin/source/hive/SchemaChecker.java |    2 +-
 39 files changed, 1894 insertions(+), 1011 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c45144b..20cb0a9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -52,12 +52,7 @@ public class CubeCapabilityChecker {
         CapabilityResult result = new CapabilityResult();
         result.capable = false;
 
-        // match joins
-        boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube);
-        if (!isJoinMatch) {
-            logger.info("Exclude cube " + cube.getName() + " because unmatched joins");
-            return result;
-        }
+        // match joins is ensured at model select
 
         // dimensions & measures
         Collection<TblColRef> dimensionColumns = getDimensionColumns(digest);
@@ -106,12 +101,12 @@ public class CubeCapabilityChecker {
         }
 
         if (!unmatchedDimensions.isEmpty()) {
-            logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions");
+            logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions: " + unmatchedDimensions);
             return result;
         }
 
         if (!unmatchedAggregations.isEmpty()) {
-            logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations");
+            logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations: " + unmatchedAggregations);
             return result;
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
deleted file mode 100644
index 9068449..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- *     http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.Collection;
-
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.realization.IRealization;
-
-public class JoinChecker {
-    
-    // given ModelChooser has done the model join matching already, this method seems useless
-    public static boolean isJoinMatch(Collection<JoinDesc> joins, IRealization realization) {
-//        List<JoinDesc> realizationsJoins = Lists.newArrayList();
-//        for (JoinTableDesc joinTable : realization.getModel().getJoinTables()) {
-//            realizationsJoins.add(joinTable.getJoin());
-//        }
-//
-//        for (JoinDesc j : joins) {
-//            // optiq engine can't decide which one is fk or pk
-//            String pTable = j.getPrimaryKeyColumns()[0].getTable();
-//            String factTable = realization.getModel().getRootFactTable().getTableIdentity();
-//            if (factTable.equals(pTable)) {
-//                j.swapPKFK();
-//            }
-//
-//            // check primary key, all PK column should refer to same tale, the Fact Table of cube.
-//            // Using first column's table name to check.
-//            String fTable = j.getForeignKeyColumns()[0].getTable();
-//            if (!factTable.equals(fTable)) {
-//                logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + realization.getName());
-//                return false;
-//            }
-//
-//            // The hashcode() function of JoinDesc has been overwritten,
-//            // which takes into consideration: pk,fk,jointype
-//            if (!realizationsJoins.contains(j)) {
-//                logger.info("Query joins don't macth on cube " + realization.getName());
-//                return false;
-//            }
-//        }
-        return true;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 05efb5e..535f77a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -132,10 +132,17 @@ public class CuboidCLI {
             while (!currentQueue.isEmpty()) {
                 long cuboid = currentQueue.pop();
                 Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
+
                 nextQueue.addAll(spnanningCuboids);
             }
             currentQueue = nextQueue;
             nextQueue = new LinkedList<Long>();
+
+            if (i == levels) {
+                if (!currentQueue.isEmpty()) {
+                    throw new IllegalStateException();
+                }
+            }
         }
 
         return allLevelCounts;

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 53c11a3..dfbe6e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -26,6 +26,7 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -887,7 +888,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
                 }
             }
         }
-        
+
         return initDimensionColRef(col);
     }
 
@@ -939,6 +940,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         for (int i = 0; i < measures.size(); i++)
             measureIndexLookup.put(measures.get(i).getName(), i);
 
+        BitSet checkEachMeasureExist = new BitSet();
         for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) {
             for (HBaseColumnDesc c : cf.getColumns()) {
                 String[] colMeasureRefs = c.getMeasureRefs();
@@ -946,13 +948,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
                 int[] measureIndex = new int[colMeasureRefs.length];
                 for (int i = 0; i < colMeasureRefs.length; i++) {
                     measureDescs[i] = measureLookup.get(colMeasureRefs[i]);
+                    checkState(measureDescs[i] != null, "measure desc at (%s) is null", i);
                     measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]);
+                    checkState(measureIndex[i] >= 0, "measure index at (%s) not positive", i);
+
+                    checkEachMeasureExist.set(measureIndex[i]);
                 }
                 c.setMeasures(measureDescs);
                 c.setMeasureIndex(measureIndex);
                 c.setColumnFamilyName(cf.getName());
             }
         }
+
+        for (int i = 0; i < measures.size(); i++) {
+            checkState(checkEachMeasureExist.get(i), "measure (%s) does not exist in column family´╝îor measure duplicates", measures.get(i));
+        }
     }
 
     private void initDictionaryDesc() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 7db616e..6457770 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -94,6 +94,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     private CubeDesc getSSBCubeDesc() {
         return getCubeDescManager().getCubeDesc("ssb");
     }
+    
+    private CubeDesc getCIInnerJoinCube() {
+        return getCubeDescManager().getCubeDesc("ci_inner_join_cube");
+    }
 
     private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
         for (long cuboidId : cuboidIds) {
@@ -219,6 +223,12 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         CubeDesc cube = getSSBCubeDesc();
         CuboidCLI.simulateCuboidGeneration(cube, true);
     }
+    
+    @Test
+    public void testCuboidGeneration7() {
+        CubeDesc cube = getCIInnerJoinCube();
+        CuboidCLI.simulateCuboidGeneration(cube, true);
+    }
 
     @Test
     public void testCuboidCounts1() {
@@ -286,6 +296,19 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     }
 
     @Test
+    public void testCuboidCounts6() {
+        CubeDesc cube = getCIInnerJoinCube();
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+        int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+        printCount(counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
+    }
+
+    @Test
     public void testCuboid_onlyBaseCuboid() {
         for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc").listFiles()) {
             if (f.getName().endsWith(".bad")) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
index a3456c4..18d84c3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.cube.model.validation.rule;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -62,10 +61,8 @@ public class FunctionRuleTest extends LocalFileMetadataTestCase {
         assertTrue(vContext.getResults().length == 0);
     }
 
-    @Test
+    @Test(expected = IllegalStateException.class)
     public void testValidateMeasureNamesDuplicated() throws IOException {
-        FunctionRule rule = new FunctionRule();
-
         File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json");
         CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
 
@@ -73,11 +70,5 @@ public class FunctionRuleTest extends LocalFileMetadataTestCase {
         desc.getMeasures().add(measureDescDuplicated);
 
         desc.init(config);
-        ValidateContext vContext = new ValidateContext();
-        rule.validate(desc, vContext);
-
-        vContext.print(System.out);
-        assertTrue(vContext.getResults().length >= 1);
-        assertEquals("There is duplicated measure's name: " + measureDescDuplicated.getName(), vContext.getResults()[0].getMessage());
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 16086e3..4665465 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -109,7 +109,7 @@ public class JoinedFlatTable {
             if (i > 0) {
                 sql.append(",");
             }
-            sql.append(col.getTableAlias() + "." + col.getName() + "\n");
+            sql.append(col.getExpressionInSourceDB() + "\n");
         }
         appendJoinStatement(flatDesc, sql);
         appendWhereStatement(flatDesc, sql);
@@ -149,7 +149,7 @@ public class JoinedFlatTable {
                         if (i > 0) {
                             sql.append(" AND ");
                         }
-                        sql.append(fk[i].getTableAlias() + "." + fk[i].getName() + " = " + pk[i].getTableAlias() + "." + pk[i].getName());
+                        sql.append(fk[i].getIdentity() + " = " + pk[i].getIdentity());
                     }
                     sql.append("\n");
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index f17983f..b3ca14a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -40,6 +41,7 @@ import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -114,6 +116,18 @@ public class MetadataManager {
     // name => External Filter Desc
     private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap;
 
+    public static class CCInfo {
+        public ComputedColumnDesc computedColumnDesc;
+        public Set<DataModelDesc> dataModelDescs;
+
+        public CCInfo(ComputedColumnDesc computedColumnDesc, Set<DataModelDesc> dataModelDescs) {
+            this.computedColumnDesc = computedColumnDesc;
+            this.dataModelDescs = dataModelDescs;
+        }
+    }
+
+    private Map<String, CCInfo> ccInfoMap = Maps.newHashMap();// this is to check any two models won't conflict computed columns
+
     private MetadataManager(KylinConfig config) throws IOException {
         init(config);
     }
@@ -343,8 +357,9 @@ public class MetadataManager {
 
         @Override
         public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
+            ccInfoMap.clear();
             for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
-                reloadDataModelDesc(model);
+                reloadDataModelDescAt(DataModelDesc.concatResourcePath(model));
             }
         }
 
@@ -353,7 +368,7 @@ public class MetadataManager {
             if (event == Event.DROP)
                 dataModelDescMap.removeLocal(cacheKey);
             else
-                reloadDataModelDesc(cacheKey);
+                reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey));
 
             for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) {
                 broadcaster.notifyProjectSchemaUpdate(prj.getName());
@@ -568,7 +583,9 @@ public class MetadataManager {
 
         List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
         for (String path : paths) {
+
             try {
+                logger.info("Reloading data model at " + path);
                 reloadDataModelDescAt(path);
             } catch (IllegalStateException e) {
                 logger.error("Error to load DataModel at " + path, e);
@@ -579,15 +596,11 @@ public class MetadataManager {
         logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)");
     }
 
-    public DataModelDesc reloadDataModelDesc(String name) {
-        return reloadDataModelDescAt(DataModelDesc.concatResourcePath(name));
-    }
-
-    private DataModelDesc reloadDataModelDescAt(String path) {
+    public DataModelDesc reloadDataModelDescAt(String path) {
         ResourceStore store = getStore();
         try {
             DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
-            dataModelDesc.init(config, this.getAllTablesMap());
+            dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
             dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
             return dataModelDesc;
         } catch (Exception e) {
@@ -635,7 +648,7 @@ public class MetadataManager {
     }
 
     private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
-        dataModelDesc.init(config, this.getAllTablesMap());
+        dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
 
         String path = dataModelDesc.getResourcePath();
         getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index e5b51e4..2cd4964 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.metadata.model;
 
 import java.io.Serializable;
+import java.util.regex.Pattern;
 
 import org.apache.kylin.metadata.datatype.DataType;
 
@@ -26,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 
 /**
  * Column Metadata from Source. All name should be uppercase.
@@ -37,10 +39,13 @@ public class ColumnDesc implements Serializable {
 
     @JsonProperty("id")
     private String id;
+
     @JsonProperty("name")
     private String name;
+
     @JsonProperty("datatype")
     private String datatype;
+
     @JsonProperty("comment")
     @JsonInclude(JsonInclude.Include.NON_NULL)
     private String comment;
@@ -62,10 +67,31 @@ public class ColumnDesc implements Serializable {
     private boolean isNullable = true;
 
     private TblColRef ref;
+    private String computedColumnExpr = null;//if null, it's not a computed column
 
     public ColumnDesc() { // default constructor for Jackson
     }
 
+    public ColumnDesc(ColumnDesc other) {
+        this.id = other.id;
+        this.name = other.name;
+        this.datatype = other.datatype;
+        this.dataGen = other.datatype;
+        this.comment = other.comment;
+        this.dataGen = other.dataGen;
+        this.index = other.index;
+    }
+
+    public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index, String computedColumnExpr) {
+        this.id = id;
+        this.name = name;
+        this.datatype = datatype;
+        this.comment = comment;
+        this.dataGen = dataGen;
+        this.index = index;
+        this.computedColumnExpr = computedColumnExpr;
+    }
+
     /** Use TableRef.getColumn() instead */
     @Deprecated
     public TblColRef getRef() {
@@ -156,7 +182,7 @@ public class ColumnDesc implements Serializable {
     public void setNullable(boolean nullable) {
         this.isNullable = nullable;
     }
-    
+
     public String getDataGen() {
         return dataGen;
     }
@@ -165,6 +191,17 @@ public class ColumnDesc implements Serializable {
         return index;
     }
 
+    public String getComputedColumnExpr(String tableAlias, String tableIdentity) {
+        Preconditions.checkState(computedColumnExpr != null);
+
+        //http://stackoverflow.com/questions/5054995/how-to-replace-case-insensitive-literal-substrings-in-java
+        return computedColumnExpr.replaceAll("(?i)" + Pattern.quote(tableIdentity), tableAlias);
+    }
+
+    public boolean isComputedColumnn() {
+        return computedColumnExpr != null;
+    }
+
     public void init(TableDesc table) {
         this.table = table;
 
@@ -212,7 +249,7 @@ public class ColumnDesc implements Serializable {
                 return false;
         } else if (!table.getIdentity().equals(other.table.getIdentity()))
             return false;
-        
+
         return true;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
new file mode 100644
index 0000000..ac5830f
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.metadata.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+
+public class ComputedColumnDesc {
+    @JsonProperty
+    private String tableIdentity;
+    @JsonProperty
+    private String columnName;
+    @JsonProperty
+    private String expression;
+    @JsonProperty
+    private String datatype;
+    @JsonProperty
+    private String comment;
+
+    public void init() {
+        Preconditions.checkNotNull(tableIdentity, "tableIdentity is null");
+        Preconditions.checkNotNull(columnName, "columnName is null");
+        Preconditions.checkNotNull(expression, "expression is null");
+        Preconditions.checkNotNull(datatype, "datatype is null");
+
+        tableIdentity = tableIdentity.toUpperCase();
+        columnName = columnName.toUpperCase();
+    }
+
+    public String getFullName() {
+        return tableIdentity + "." + columnName;
+    }
+
+    public String getTableIdentity() {
+        return tableIdentity;
+    }
+
+    public String getColumnName() {
+        return columnName;
+    }
+
+    public String getExpression() {
+        return expression;
+    }
+
+    public String getDatatype() {
+        return datatype;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        ComputedColumnDesc that = (ComputedColumnDesc) o;
+
+        if (!tableIdentity.equals(that.tableIdentity))
+            return false;
+        if (!columnName.equals(that.columnName))
+            return false;
+        if (!expression.equals(that.expression))
+            return false;
+        return datatype.equals(that.datatype);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = tableIdentity.hashCode();
+        result = 31 * result + columnName.hashCode();
+        result = 31 * result + expression.hashCode();
+        result = 31 * result + datatype.hashCode();
+        return result;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 3669109..2174b7a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.metadata.model;
 
+import static org.apache.kylin.metadata.MetadataManager.CCInfo;
+
 import java.io.Serializable;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -28,6 +30,9 @@ import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.mutable.MutableInt;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
@@ -42,6 +47,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -96,6 +104,10 @@ public class DataModelDesc extends RootPersistentEntity {
     @JsonProperty("capacity")
     private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
 
+    @JsonProperty("computed_columns")
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    private List<ComputedColumnDesc> computedColumnDescs = Lists.newArrayList();
+
     // computed attributes
     private TableRef rootFactTableRef;
     private Set<TableRef> factTableRefs = Sets.newLinkedHashSet();
@@ -291,7 +303,16 @@ public class DataModelDesc extends RootPersistentEntity {
         throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name);
     }
 
-    public void init(KylinConfig config, Map<String, TableDesc> tables) {
+    public void init(KylinConfig config, Map<String, TableDesc> originalTables, Map<String, CCInfo> ccInfoMap) {
+        //tweak the tables according to Computed Columns defined in model
+        Map<String, TableDesc> tables = Maps.newHashMap();
+        for (Map.Entry<String, TableDesc> entry : originalTables.entrySet()) {
+            String s = entry.getKey();
+            TableDesc tableDesc = entry.getValue();
+            TableDesc extendedTableDesc = tableDesc.appendColumns(createComputedColumns(tableDesc));
+            tables.put(s, extendedTableDesc);
+        }
+
         this.config = config;
 
         initJoinTablesForUpgrade();
@@ -301,13 +322,32 @@ public class DataModelDesc extends RootPersistentEntity {
         initJoinsTree();
         initDimensionsAndMetrics();
         initPartitionDesc();
+        initComputedColumns(ccInfoMap);
 
         boolean reinit = validate();
         if (reinit) { // model slightly changed by validate() and must init() again
-            init(config, tables);
+            init(config, tables, ccInfoMap);
         }
     }
 
+    private ColumnDesc[] createComputedColumns(final TableDesc tableDesc) {
+        final MutableInt id = new MutableInt(tableDesc.getColumnCount());
+        return FluentIterable.from(this.computedColumnDescs).filter(new Predicate<ComputedColumnDesc>() {
+            @Override
+            public boolean apply(@Nullable ComputedColumnDesc input) {
+                return tableDesc.getIdentity().equalsIgnoreCase(input.getTableIdentity());
+            }
+        }).transform(new Function<ComputedColumnDesc, ColumnDesc>() {
+            @Nullable
+            @Override
+            public ColumnDesc apply(@Nullable ComputedColumnDesc input) {
+                id.increment();
+                ColumnDesc columnDesc = new ColumnDesc(id.toString(), input.getColumnName(), input.getDatatype(), input.getComment(), null, null, input.getExpression());
+                return columnDesc;
+            }
+        }).toArray(ColumnDesc.class);
+    }
+
     private void initJoinTablesForUpgrade() {
         if (joinTables == null) {
             joinTables = new JoinTableDesc[0];
@@ -333,6 +373,7 @@ public class DataModelDesc extends RootPersistentEntity {
 
         TableDesc rootDesc = tables.get(rootFactTable);
         rootFactTableRef = new TableRef(this, rootDesc.getName(), rootDesc);
+
         addAlias(rootFactTableRef);
         factTableRefs.add(rootFactTableRef);
 
@@ -346,7 +387,9 @@ public class DataModelDesc extends RootPersistentEntity {
             String alias = join.getAlias();
             if (alias == null)
                 alias = tableDesc.getName();
+
             TableRef ref = new TableRef(this, alias, tableDesc);
+
             join.setTableRef(ref);
             addAlias(ref);
             (join.getKind() == TableKind.LOOKUP ? lookupTableRefs : factTableRefs).add(ref);
@@ -390,6 +433,30 @@ public class DataModelDesc extends RootPersistentEntity {
             this.partitionDesc.init(this);
     }
 
+    private void initComputedColumns(Map<String, CCInfo> ccInfoMap) {
+        Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model
+
+        for (ComputedColumnDesc computedColumnDesc : this.computedColumnDescs) {
+            computedColumnDesc.init();
+
+            if (ccSet.contains(computedColumnDesc.getFullName())) {
+                throw new IllegalArgumentException(String.format("More than one computed column named %s exist in model %s", computedColumnDesc.getFullName(), this.getName()));
+            } else {
+                ccSet.add(computedColumnDesc.getFullName());
+            }
+
+            CCInfo other = ccInfoMap.get(computedColumnDesc.getFullName());
+            if (other == null || (other.dataModelDescs.size() == 1 && other.dataModelDescs.contains(this))) {
+                ccInfoMap.put(computedColumnDesc.getFullName(), new CCInfo(computedColumnDesc, Sets.<DataModelDesc> newHashSet(this)));
+            } else if (other.computedColumnDesc.equals(computedColumnDesc)) {
+                other.dataModelDescs.add(this);
+            } else {
+                throw new IllegalStateException(String.format("Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", //
+                        computedColumnDesc.getFullName(), other.dataModelDescs));
+            }
+        }
+    }
+
     private void initJoinColumns() {
 
         for (JoinTableDesc joinTable : joinTables) {
@@ -568,10 +635,8 @@ public class DataModelDesc extends RootPersistentEntity {
     }
 
     /**
-     * @param message
-     *            error message
-     * @param silent
-     *            if throw exception
+     * @param message error message
+     * @param silent  if throw exception
      */
     public void addError(String message, boolean silent) {
         if (!silent) {
@@ -627,6 +692,10 @@ public class DataModelDesc extends RootPersistentEntity {
         return dimensions;
     }
 
+    public List<ComputedColumnDesc> getComputedColumnDescs() {
+        return computedColumnDescs;
+    }
+
     public String[] getMetrics() {
         return metrics;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 8e2192f..7db730e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -93,7 +93,7 @@ public class JoinsTree implements Serializable {
 
         boolean matches = false;
         if (chain.join == null) {
-            matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc());
+            matches = anotherChain.join == null && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity());
         } else {
             matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 38c2de5..3c00149 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -201,7 +201,7 @@ public class PartitionDesc implements Serializable {
         }
 
         private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) {
-            String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName();
+            String partitionColumnName = partitionColumn.getIdentity();
             if (startInclusive > 0) {
                 builder.append(partitionColumnName + " >= " + startInclusive);
                 builder.append(" AND ");
@@ -210,7 +210,7 @@ public class PartitionDesc implements Serializable {
         }
 
         private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) {
-            String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName();
+            String partitionColumnName = partitionColumn.getIdentity();
             if (startInclusive > 0) {
                 builder.append(partitionColumnName + " >= " + DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN));
                 builder.append(" AND ");
@@ -219,7 +219,7 @@ public class PartitionDesc implements Serializable {
         }
 
         private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat) {
-            String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName();
+            String partitionColumnName = partitionColumn.getIdentity();
             if (startInclusive > 0) {
                 builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'");
                 builder.append(" AND ");
@@ -228,8 +228,8 @@ public class PartitionDesc implements Serializable {
         }
 
         private static void buildMultipleColumnRangeCondition(StringBuilder builder, TblColRef partitionDateColumn, TblColRef partitionTimeColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat, String partitionColumnTimeFormat) {
-            String partitionDateColumnName = partitionDateColumn.getTableAlias() + "." + partitionDateColumn.getName();
-            String partitionTimeColumnName = partitionTimeColumn.getTableAlias() + "." + partitionTimeColumn.getName();
+            String partitionDateColumnName = partitionDateColumn.getIdentity();
+            String partitionTimeColumnName = partitionTimeColumn.getIdentity();
             if (startInclusive > 0) {
                 builder.append("(");
                 builder.append("(");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index b4d49fb..b388f11 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -61,10 +61,34 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
     }
 
     public TableDesc(TableDesc other) {
-        this.name = other.getName();
-        this.columns = other.getColumns();
+        this.name = other.name;
+        this.sourceType = other.sourceType;
         this.database.setName(other.getDatabase());
-        this.tableType = other.getTableType();
+        this.tableType = other.tableType;
+        this.dataGen = other.dataGen;
+        this.columns = new ColumnDesc[other.columns.length];
+        for (int i = 0; i < other.columns.length; i++) {
+            this.columns[i] = new ColumnDesc(other.columns[i]);
+            this.columns[i].init(this);
+        }
+    }
+
+    public TableDesc appendColumns(ColumnDesc[] computedColumns) {
+        if (computedColumns == null || computedColumns.length == 0) {
+            return this;
+        }
+
+        TableDesc ret = new TableDesc(this);//deep copy of the table desc
+        ColumnDesc[] origin = ret.columns;
+        ret.columns = new ColumnDesc[computedColumns.length + origin.length];
+        for (int i = 0; i < origin.length; i++) {
+            ret.columns[i] = origin[i];
+        }
+        for (int i = 0; i < computedColumns.length; i++) {
+            computedColumns[i].init(ret);
+            ret.columns[i + this.columns.length] = computedColumns[i];
+        }
+        return ret;
     }
 
     public ColumnDesc findColumnByName(String name) {
@@ -153,7 +177,12 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
     }
 
     public int getMaxColumnIndex() {
+        if (columns == null) {
+            return -1;
+        }
+        
         int max = -1;
+        
         for (ColumnDesc col : columns) {
             int idx = col.getZeroBasedIndex();
             max = Math.max(max, idx);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 6f15f3c..f4aec12 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -18,14 +18,14 @@
 
 package org.apache.kylin.metadata.model;
 
-import com.google.common.collect.Maps;
-
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 
-public class TableRef implements Serializable{
+import com.google.common.collect.Maps;
+
+public class TableRef implements Serializable {
 
     final transient private DataModelDesc model;
     final private String alias;
@@ -68,11 +68,11 @@ public class TableRef implements Serializable{
     public TblColRef getColumn(String name) {
         return columns.get(name);
     }
-    
+
     public Collection<TblColRef> getColumns() {
         return Collections.unmodifiableCollection(columns.values());
     }
-    
+
     // for test only
     @Deprecated
     public TblColRef makeFakeColumn(String name) {
@@ -81,13 +81,13 @@ public class TableRef implements Serializable{
         colDesc.setTable(table);
         return new TblColRef(this, colDesc);
     }
-    
+
     // for test only
     @Deprecated
     public TblColRef makeFakeColumn(ColumnDesc colDesc) {
         return new TblColRef(this, colDesc);
     }
-    
+
     @Override
     public boolean equals(Object o) {
         if (this == o)

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 4661bc9..da62a75 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -56,7 +56,7 @@ public class TblColRef implements Serializable {
     public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType) {
         return newInnerColumn(columnName, dataType, null);
     }
-    
+
     // used by projection rewrite, see OLAPProjectRel
     public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType, String parserDescription) {
         ColumnDesc column = new ColumnDesc();
@@ -68,25 +68,25 @@ public class TblColRef implements Serializable {
         colRef.parserDescription = parserDescription;
         return colRef;
     }
-    
+
     private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc();
     static {
         UNKNOWN_MODEL.setName("UNKNOWN_MODEL");
     }
-    
+
     public static TableRef tableForUnknownModel(String tempTableAlias, TableDesc table) {
         return new TableRef(UNKNOWN_MODEL, tempTableAlias, table);
     }
-    
+
     public static TblColRef columnForUnknownModel(TableRef table, ColumnDesc colDesc) {
         checkArgument(table.getModel() == UNKNOWN_MODEL);
         return new TblColRef(table, colDesc);
     }
-    
+
     public static void fixUnknownModel(DataModelDesc model, String alias, TblColRef col) {
         checkArgument(col.table.getModel() == UNKNOWN_MODEL || col.table.getModel() == model);
         TableRef tableRef = model.findTable(alias);
-        checkArgument(tableRef.getTableDesc() == col.column.getTable());
+        checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity()));
         col.table = tableRef;
         col.identity = null;
     }
@@ -101,7 +101,7 @@ public class TblColRef implements Serializable {
         desc.init(table);
         return new TblColRef(desc);
     }
-    
+
     // ============================================================================
 
     private TableRef table;
@@ -112,13 +112,13 @@ public class TblColRef implements Serializable {
     TblColRef(ColumnDesc column) {
         this.column = column;
     }
-    
+
     TblColRef(TableRef table, ColumnDesc column) {
-        checkArgument(table.getTableDesc() == column.getTable());
+        checkArgument(table.getTableDesc().getIdentity().equals(column.getTable().getIdentity()));
         this.table = table;
         this.column = column;
     }
-    
+
     public ColumnDesc getColumnDesc() {
         return column;
     }
@@ -130,15 +130,23 @@ public class TblColRef implements Serializable {
     public TableRef getTableRef() {
         return table;
     }
-    
+
     public boolean isQualified() {
         return table != null;
     }
-    
+
     public String getTableAlias() {
         return table != null ? table.getAlias() : "UNKNOWN_ALIAS";
     }
-    
+
+    public String getExpressionInSourceDB() {
+        if (!column.isComputedColumnn()) {
+            return getIdentity();
+        } else {
+            return column.getComputedColumnExpr(getTableAlias(), table.getTableIdentity());
+        }
+    }
+
     public String getTable() {
         if (column.getTable() == null) {
             return null;
@@ -207,7 +215,7 @@ public class TblColRef implements Serializable {
     public String toString() {
         if (isInnerColumn() && parserDescription != null)
             return parserDescription;
-        
+
         String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias();
         String tableName = column.getTable() == null ? "NULL" : column.getTable().getName();
         String tableIdentity = column.getTable() == null ? "NULL" : column.getTable().getIdentity();

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 63188a1..deac12d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -242,10 +242,15 @@ class ProjectL2Cache {
                 logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager");
                 return false;
             }
-            ColumnDesc foundCol = table.findColumnByName(col.getName());
-            if (col.getColumnDesc().equals(foundCol) == false) {
-                logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
-                return false;
+
+            if (!col.getColumnDesc().isComputedColumnn()) {
+                ColumnDesc foundCol = table.findColumnByName(col.getName());
+                if (col.getColumnDesc().equals(foundCol) == false) {
+                    logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
+                    return false;
+                }
+            } else {
+                //computed column may not exit here
             }
 
             // auto-define table required by realization for some legacy test case

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
index b319423..f6d6fc3 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.metadata.model;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.commons.lang.ArrayUtils;
@@ -41,7 +42,19 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
     public void after() throws Exception {
         this.cleanupTestMetadata();
     }
-    
+
+    @Test
+    public void loadInnerModel() {
+        DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_inner_join_model");
+        assertNotNull(model);
+    }
+
+    @Test
+    public void loadLeftModel() {
+        DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_left_join_model");
+        assertNotNull(model);
+    }
+
     @Test
     public void testNoDupColInDimAndMeasure() {
         DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index cfaadac..3747af8 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -1,445 +1,615 @@
 {
-  "uuid" : "3819ad72-3929-4dff-b59d-cd89a01238af",
-  "name" : "ci_inner_join_cube",
-  "model_name" : "ci_inner_join_model",
-  "description" : null,
-  "dimensions" : [ {
-    "name" : "CAL_DT",
-    "table" : "TEST_CAL_DT",
-    "column" : "{FK}",
-    "derived" : [ "WEEK_BEG_DT" ]
-  }, {
-    "name" : "ORDER_ID",
-    "table" : "TEST_KYLIN_FACT",
-    "column" : "ORDER_ID"
-  }, {
-    "name" : "TEST_DATE_ENC",
-    "table" : "TEST_ORDER",
-    "column" : "TEST_DATE_ENC"
-  }, {
-    "name" : "TEST_TIME_ENC",
-    "table" : "TEST_ORDER",
-    "column" : "TEST_TIME_ENC"
-  }, {
-    "name" : "CATEGORY",
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : "{FK}",
-    "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
-  }, {
-    "name" : "CATEGORY_HIERARCHY",
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : "META_CATEG_NAME",
-    "derived" : null
-  }, {
-    "name" : "CATEGORY_HIERARCHY",
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : "CATEG_LVL2_NAME",
-    "derived" : null
-  }, {
-    "name" : "CATEGORY_HIERARCHY",
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : "CATEG_LVL3_NAME",
-    "derived" : null
-  }, {
-    "name" : "LSTG_FORMAT_NAME",
-    "table" : "TEST_KYLIN_FACT",
-    "column" : "LSTG_FORMAT_NAME",
-    "derived" : null
-  }, {
-    "name" : "SITE_ID",
-    "table" : "TEST_SITES",
-    "column" : "{FK}",
-    "derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-    "name" : "SELLER_TYPE_CD",
-    "table" : "TEST_SELLER_TYPE_DIM",
-    "column" : "{FK}",
-    "derived" : [ "SELLER_TYPE_DESC" ]
-  }, {
-    "name" : "SELLER_ID",
-    "table" : "TEST_KYLIN_FACT",
-    "column" : "SELLER_ID"
-  }, {
-    "name" : "SELLER_BUYER_LEVEL",
-    "table" : "SELLER_ACCOUNT",
-    "column" : "ACCOUNT_BUYER_LEVEL"
-  }, {
-    "name" : "SELLER_SELLER_LEVEL",
-    "table" : "SELLER_ACCOUNT",
-    "column" : "ACCOUNT_SELLER_LEVEL"
-  }, {
-    "name" : "SELLER_COUNTRY",
-    "table" : "SELLER_ACCOUNT",
-    "column" : "ACCOUNT_COUNTRY"
-  }, {
-    "name" : "SELLER_COUNTRY_NAME",
-    "table" : "SELLER_COUNTRY",
-    "column" : "NAME"
-  }, {
-    "name" : "BUYER_ID",
-    "table" : "TEST_ORDER",
-    "column" : "BUYER_ID"
-  }, {
-    "name" : "BUYER_BUYER_LEVEL",
-    "table" : "BUYER_ACCOUNT",
-    "column" : "ACCOUNT_BUYER_LEVEL"
-  }, {
-    "name" : "BUYER_SELLER_LEVEL",
-    "table" : "BUYER_ACCOUNT",
-    "column" : "ACCOUNT_SELLER_LEVEL"
-  }, {
-    "name" : "BUYER_COUNTRY",
-    "table" : "BUYER_ACCOUNT",
-    "column" : "ACCOUNT_COUNTRY"
-  }, {
-    "name" : "BUYER_COUNTRY_NAME",
-    "table" : "BUYER_COUNTRY",
-    "column" : "NAME"
-  } ],
-  "measures" : [ {
-    "name" : "TRANS_CNT",
-    "function" : {
-      "expression" : "COUNT",
-      "parameter" : {
-        "type" : "constant",
-        "value" : "1"
-      },
-      "returntype" : "bigint"
+  "uuid": "3819ad72-3929-4dff-b59d-cd89a01238af",
+  "name": "ci_inner_join_cube",
+  "model_name": "ci_inner_join_model",
+  "description": null,
+  "dimensions": [
+    {
+      "name": "CAL_DT",
+      "table": "TEST_CAL_DT",
+      "column": "{FK}",
+      "derived": [
+        "WEEK_BEG_DT"
+      ]
+    },
+    {
+      "name": "ORDER_ID",
+      "table": "TEST_KYLIN_FACT",
+      "column": "ORDER_ID"
+    },
+    {
+      "name": "TEST_DATE_ENC",
+      "table": "TEST_ORDER",
+      "column": "TEST_DATE_ENC"
+    },
+    {
+      "name": "TEST_TIME_ENC",
+      "table": "TEST_ORDER",
+      "column": "TEST_TIME_ENC"
+    },
+    {
+      "name": "CATEGORY",
+      "table": "TEST_CATEGORY_GROUPINGS",
+      "column": "{FK}",
+      "derived": [
+        "USER_DEFINED_FIELD1",
+        "USER_DEFINED_FIELD3",
+        "UPD_DATE",
+        "UPD_USER"
+      ]
+    },
+    {
+      "name": "CATEGORY_HIERARCHY",
+      "table": "TEST_CATEGORY_GROUPINGS",
+      "column": "META_CATEG_NAME",
+      "derived": null
+    },
+    {
+      "name": "CATEGORY_HIERARCHY",
+      "table": "TEST_CATEGORY_GROUPINGS",
+      "column": "CATEG_LVL2_NAME",
+      "derived": null
+    },
+    {
+      "name": "CATEGORY_HIERARCHY",
+      "table": "TEST_CATEGORY_GROUPINGS",
+      "column": "CATEG_LVL3_NAME",
+      "derived": null
+    },
+    {
+      "name": "LSTG_FORMAT_NAME",
+      "table": "TEST_KYLIN_FACT",
+      "column": "LSTG_FORMAT_NAME",
+      "derived": null
+    },
+    {
+      "name": "SITE_ID",
+      "table": "TEST_SITES",
+      "column": "{FK}",
+      "derived": [
+        "SITE_NAME",
+        "CRE_USER"
+      ]
+    },
+    {
+      "name": "SELLER_TYPE_CD",
+      "table": "TEST_SELLER_TYPE_DIM",
+      "column": "{FK}",
+      "derived": [
+        "SELLER_TYPE_DESC"
+      ]
+    },
+    {
+      "name": "SELLER_ID",
+      "table": "TEST_KYLIN_FACT",
+      "column": "SELLER_ID"
+    },
+    {
+      "name": "SELLER_BUYER_LEVEL",
+      "table": "SELLER_ACCOUNT",
+      "column": "ACCOUNT_BUYER_LEVEL"
+    },
+    {
+      "name": "SELLER_SELLER_LEVEL",
+      "table": "SELLER_ACCOUNT",
+      "column": "ACCOUNT_SELLER_LEVEL"
+    },
+    {
+      "name": "SELLER_COUNTRY",
+      "table": "SELLER_ACCOUNT",
+      "column": "ACCOUNT_COUNTRY"
+    },
+    {
+      "name": "SELLER_COUNTRY_NAME",
+      "table": "SELLER_COUNTRY",
+      "column": "NAME"
+    },
+    {
+      "name": "BUYER_ID",
+      "table": "TEST_ORDER",
+      "column": "BUYER_ID"
+    },
+    {
+      "name": "BUYER_BUYER_LEVEL",
+      "table": "BUYER_ACCOUNT",
+      "column": "ACCOUNT_BUYER_LEVEL"
+    },
+    {
+      "name": "BUYER_SELLER_LEVEL",
+      "table": "BUYER_ACCOUNT",
+      "column": "ACCOUNT_SELLER_LEVEL"
+    },
+    {
+      "name": "BUYER_COUNTRY",
+      "table": "BUYER_ACCOUNT",
+      "column": "ACCOUNT_COUNTRY"
+    },
+    {
+      "name": "DEAL_YEAR",
+      "table": "TEST_KYLIN_FACT",
+      "column": "DEAL_YEAR"
+    },
+    {
+      "name": "SELLER_COUNTRY_ABBR",
+      "table": "SELLER_ACCOUNT",
+      "column": "COUNTRY_ABBR"
+    },
+    {
+      "name": "BUYER_COUNTRY_ABBR",
+      "table": "BUYER_ACCOUNT",
+      "column": "COUNTRY_ABBR"
+    },
+    {
+      "name": "BUYER_COUNTRY_NAME",
+      "table": "BUYER_COUNTRY",
+      "column": "NAME"
     }
-  }, {
-    "name" : "ITEM_COUNT_SUM",
-    "function" : {
-      "expression" : "SUM",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.ITEM_COUNT"
-      },
-      "returntype" : "bigint"
+  ],
+  "measures": [
+    {
+      "name": "TRANS_CNT",
+      "function": {
+        "expression": "COUNT",
+        "parameter": {
+          "type": "constant",
+          "value": "1"
+        },
+        "returntype": "bigint"
+      }
+    },
+    {
+      "name": "ITEM_COUNT_SUM",
+      "function": {
+        "expression": "SUM",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.ITEM_COUNT"
+        },
+        "returntype": "bigint"
+      }
+    },
+    {
+      "name": "GMV_SUM",
+      "function": {
+        "expression": "SUM",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      }
+    },
+    {
+      "name": "GMV_MIN",
+      "function": {
+        "expression": "MIN",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      }
+    },
+    {
+      "name": "GMV_MAX",
+      "function": {
+        "expression": "MAX",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      }
+    },
+    {
+      "name": "SELLER_HLL",
+      "function": {
+        "expression": "COUNT_DISTINCT",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.SELLER_ID"
+        },
+        "returntype": "hllc(10)"
+      }
+    },
+    {
+      "name": "SELLER_FORMAT_HLL",
+      "function": {
+        "expression": "COUNT_DISTINCT",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+          "next_parameter": {
+            "type": "column",
+            "value": "TEST_KYLIN_FACT.SELLER_ID"
+          }
+        },
+        "returntype": "hllc(10)"
+      }
+    },
+    {
+      "name": "TOP_SELLER",
+      "function": {
+        "expression": "TOP_N",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.PRICE",
+          "next_parameter": {
+            "type": "column",
+            "value": "TEST_KYLIN_FACT.SELLER_ID"
+          }
+        },
+        "returntype": "topn(100, 4)",
+        "configuration": {
+          "topn.encoding.TEST_KYLIN_FACT.SELLER_ID": "int:4"
+        }
+      }
+    },
+    {
+      "name": "TEST_COUNT_DISTINCT_BITMAP",
+      "function": {
+        "expression": "COUNT_DISTINCT",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
+        },
+        "returntype": "bitmap"
+      }
+    },
+    {
+      "name": "TEST_EXTENDED_COLUMN",
+      "function": {
+        "expression": "EXTENDED_COLUMN",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.ORDER_ID",
+          "next_parameter": {
+            "type": "column",
+            "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
+          }
+        },
+        "returntype": "extendedcolumn(100)"
+      }
+    },
+    {
+      "name": "BUYER_CONTACT",
+      "function": {
+        "expression": "EXTENDED_COLUMN",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_ORDER.BUYER_ID",
+          "next_parameter": {
+            "type": "column",
+            "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
+          }
+        },
+        "returntype": "extendedcolumn(100)"
+      }
+    },
+    {
+      "name": "SELLER_CONTACT",
+      "function": {
+        "expression": "EXTENDED_COLUMN",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.SELLER_ID",
+          "next_parameter": {
+            "type": "column",
+            "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
+          }
+        },
+        "returntype": "extendedcolumn(100)"
+      }
+    },
+    {
+      "name": "TRANS_ID_RAW",
+      "function": {
+        "expression": "RAW",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.TRANS_ID"
+        },
+        "returntype": "raw"
+      }
+    },
+    {
+      "name": "PRICE_RAW",
+      "function": {
+        "expression": "RAW",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.PRICE"
+        },
+        "returntype": "raw"
+      }
+    },
+    {
+      "name": "CAL_DT_RAW",
+      "function": {
+        "expression": "RAW",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.CAL_DT"
+        },
+        "returntype": "raw"
+      }
+    },
+    {
+      "name": "COMPUTED_COLUMN_MEASURE",
+      "function": {
+        "expression": "SUM",
+        "parameter": {
+          "type": "column",
+          "value": "TEST_KYLIN_FACT.DEAL_AMOUNT"
+        },
+        "returntype": "decimal"
+      }
     }
-  }, {
-    "name" : "GMV_SUM",
-    "function" : {
-      "expression" : "SUM",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.PRICE"
-      },
-      "returntype" : "decimal(19,4)"
+  ],
+  "dictionaries": [
+    {
+      "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
+      "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
     }
-  }, {
-    "name" : "GMV_MIN",
-    "function" : {
-      "expression" : "MIN",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.PRICE"
+  ],
+  "rowkey": {
+    "rowkey_columns": [
+      {
+        "column": "TEST_KYLIN_FACT.SELLER_ID",
+        "encoding": "int:4"
       },
-      "returntype" : "decimal(19,4)"
-    }
-  }, {
-    "name" : "GMV_MAX",
-    "function" : {
-      "expression" : "MAX",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.PRICE"
+      {
+        "column": "TEST_KYLIN_FACT.ORDER_ID",
+        "encoding": "int:4"
       },
-      "returntype" : "decimal(19,4)"
-    }
-  }, {
-    "name" : "SELLER_HLL",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.SELLER_ID"
+      {
+        "column": "TEST_KYLIN_FACT.CAL_DT",
+        "encoding": "dict"
       },
-      "returntype" : "hllc(10)"
-    }
-  }, {
-    "name" : "SELLER_FORMAT_HLL",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
-        "next_parameter" : {
-          "type" : "column",
-          "value" : "TEST_KYLIN_FACT.SELLER_ID"
-        }
+      {
+        "column": "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+        "encoding": "dict"
       },
-      "returntype" : "hllc(10)"
-    }
-  }, {
-    "name" : "TOP_SELLER",
-    "function" : {
-      "expression" : "TOP_N",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.PRICE",
-        "next_parameter" : {
-          "type" : "column",
-          "value" : "TEST_KYLIN_FACT.SELLER_ID"
-        }
+      {
+        "column": "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+        "encoding": "dict"
       },
-      "returntype" : "topn(100, 4)",
-      "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"}
-    }
-  }, {
-    "name" : "TEST_COUNT_DISTINCT_BITMAP",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
+      {
+        "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+        "encoding": "dict"
       },
-      "returntype" : "bitmap"
-    }
-  }, {
-    "name" : "TEST_EXTENDED_COLUMN",
-    "function" : {
-      "expression" : "EXTENDED_COLUMN",
-      "parameter": {
-        "type": "column",
-        "value": "TEST_KYLIN_FACT.ORDER_ID",
-        "next_parameter": {
-          "type": "column",
-          "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
-        }
+      {
+        "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+        "encoding": "dict"
       },
-      "returntype": "extendedcolumn(100)"
-    }
-  }, {
-    "name" : "BUYER_CONTACT",
-    "function" : {
-      "expression" : "EXTENDED_COLUMN",
-      "parameter": {
-        "type": "column",
-        "value": "TEST_ORDER.BUYER_ID",
-        "next_parameter": {
-          "type": "column",
-          "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
-        }
+      {
+        "column": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+        "encoding": "fixed_length:12"
       },
-      "returntype": "extendedcolumn(100)"
-    }
-  }, {
-    "name" : "SELLER_CONTACT",
-    "function" : {
-      "expression" : "EXTENDED_COLUMN",
-      "parameter": {
-        "type": "column",
-        "value": "TEST_KYLIN_FACT.SELLER_ID",
-        "next_parameter": {
-          "type": "column",
-          "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
-        }
+      {
+        "column": "TEST_KYLIN_FACT.LSTG_SITE_ID",
+        "encoding": "dict"
       },
-      "returntype": "extendedcolumn(100)"
-    }
-  }, {
-    "name" : "TRANS_ID_RAW",
-    "function" : {
-      "expression" : "RAW",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.TRANS_ID"
+      {
+        "column": "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+        "encoding": "dict"
       },
-      "returntype" : "raw"
-    }
-  }, {
-    "name" : "PRICE_RAW",
-    "function" : {
-      "expression" : "RAW",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.PRICE"
+      {
+        "column": "TEST_ORDER.TEST_TIME_ENC",
+        "encoding": "time"
       },
-      "returntype" : "raw"
-    }
-  }, {
-    "name" : "CAL_DT_RAW",
-    "function" : {
-      "expression" : "RAW",
-      "parameter" : {
-        "type" : "column",
-        "value" : "TEST_KYLIN_FACT.CAL_DT"
+      {
+        "column": "TEST_ORDER.TEST_DATE_ENC",
+        "encoding": "date"
       },
-      "returntype" : "raw"
-    }
-  } ],
-  "dictionaries": [ {
-    "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
-    "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
-  } ],
-  "rowkey" : {
-    "rowkey_columns" : [ {
-      "column" : "TEST_KYLIN_FACT.SELLER_ID",
-      "encoding" : "int:4"
-    }, {
-      "column" : "TEST_KYLIN_FACT.ORDER_ID",
-      "encoding" : "int:4"
-    }, {
-      "column" : "TEST_KYLIN_FACT.CAL_DT",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
-      "encoding" : "fixed_length:12"
-    }, {
-      "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
-      "encoding" : "dict"
-    }, {
-      "column" : "TEST_ORDER.TEST_TIME_ENC",
-      "encoding" : "time"
-    }, {
-      "column" : "TEST_ORDER.TEST_DATE_ENC",
-      "encoding" : "date"
-    }, {
-      "column" : "TEST_ORDER.BUYER_ID",
-      "encoding" : "int:4"
-    }, {
-      "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-      "encoding" : "dict"
-    }, {
-      "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-      "encoding" : "dict"
-    }, {
-      "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
-      "encoding" : "dict"
-    }, {
-      "column" : "BUYER_COUNTRY.NAME",
-      "encoding" : "dict"
-    }, {
-      "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-      "encoding" : "dict"
-    }, {
-      "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-      "encoding" : "dict"
-    }, {
-      "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
-      "encoding" : "dict"
-    }, {
-      "column" : "SELLER_COUNTRY.NAME",
-      "encoding" : "dict"
-    } ]
+      {
+        "column": "TEST_ORDER.BUYER_ID",
+        "encoding": "int:4"
+      },
+      {
+        "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+        "encoding": "dict"
+      },
+      {
+        "column": "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+        "encoding": "dict"
+      },
+      {
+        "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+        "encoding": "dict"
+      },
+      {
+        "column": "BUYER_COUNTRY.NAME",
+        "encoding": "dict"
+      },
+      {
+        "column": "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+        "encoding": "dict"
+      },
+      {
+        "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+        "encoding": "dict"
+      },
+      {
+        "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+        "encoding": "dict"
+      },
+      {
+        "column": "TEST_KYLIN_FACT.DEAL_YEAR",
+        "encoding": "int:2"
+      },
+      {
+        "column": "SELLER_COUNTRY.NAME",
+        "encoding": "dict"
+      },
+      {
+        "column": "SELLER_ACCOUNT.COUNTRY_ABBR",
+        "encoding": "dict"
+      },
+      {
+        "column": "BUYER_ACCOUNT.COUNTRY_ABBR",
+        "encoding": "dict"
+      }
+    ]
   },
-  "signature" : null,
-  "last_modified" : 1448959801311,
-  "null_string" : null,
-  "hbase_mapping" : {
-    "column_family" : [ {
-      "name" : "f1",
-      "columns" : [ {
-        "qualifier" : "m",
-        "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ]
-      } ]
-    }, {
-      "name" : "f2",
-      "columns" : [ {
-        "qualifier" : "m",
-        "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ]
-      } ]
-    }, {
-      "name" : "f3",
-      "columns" : [ {
-        "qualifier" : "m",
-        "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ]
-      } ]
-    } ]
+  "signature": null,
+  "last_modified": 1448959801311,
+  "null_string": null,
+  "hbase_mapping": {
+    "column_family": [
+      {
+        "name": "f1",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "TRANS_CNT",
+              "ITEM_COUNT_SUM",
+              "GMV_SUM",
+              "GMV_MIN",
+              "GMV_MAX",
+              "COMPUTED_COLUMN_MEASURE"
+            ]
+          }
+        ]
+      },
+      {
+        "name": "f2",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "SELLER_HLL",
+              "SELLER_FORMAT_HLL",
+              "TOP_SELLER",
+              "TEST_COUNT_DISTINCT_BITMAP"
+            ]
+          }
+        ]
+      },
+      {
+        "name": "f3",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "TEST_EXTENDED_COLUMN",
+              "TRANS_ID_RAW",
+              "PRICE_RAW",
+              "CAL_DT_RAW",
+              "BUYER_CONTACT",
+              "SELLER_CONTACT"
+            ]
+          }
+        ]
+      }
+    ]
   },
-  "aggregation_groups" : [ {
-    "includes" : [ "TEST_KYLIN_FACT.CAL_DT", 
-                   "TEST_KYLIN_FACT.LEAF_CATEG_ID", 
-                   "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", 
-                   "TEST_KYLIN_FACT.LSTG_SITE_ID", 
-                   "TEST_KYLIN_FACT.SLR_SEGMENT_CD", 
-                   "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", 
-                   "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 
-                   "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ],
-    "select_rule" : {
-      "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", 
-                             "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 
-                             "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 
-                             "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ],
-      "mandatory_dims" : [ ],
-      "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", 
-                         "TEST_KYLIN_FACT.LSTG_SITE_ID", 
-                         "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ]
-    }
-  }, {
-    "includes" : [ "TEST_KYLIN_FACT.CAL_DT", 
-                   "TEST_KYLIN_FACT.LEAF_CATEG_ID", 
-                   "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", 
-                   "TEST_KYLIN_FACT.LSTG_SITE_ID", 
-                   "TEST_KYLIN_FACT.SLR_SEGMENT_CD", 
-                   "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", 
-                   "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 
-                   "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 
-                   
-                   "TEST_KYLIN_FACT.SELLER_ID", 
-                   "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-                   "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-                   "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
-                   "SELLER_COUNTRY.NAME",
-                   
-                   "TEST_KYLIN_FACT.ORDER_ID", 
-                   "TEST_ORDER.TEST_DATE_ENC", 
-                   "TEST_ORDER.TEST_TIME_ENC",
-                   "TEST_ORDER.BUYER_ID",
-                   "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-                   "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-                   "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
-                   "BUYER_COUNTRY.NAME" ],
-    "select_rule" : {
-      "hierarchy_dims" : [ ],
-      "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ],
-      
-      "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", 
-                         "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 
-                         "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 
-                         "TEST_KYLIN_FACT.LEAF_CATEG_ID" ], 
-                       
-                       [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", 
-                         "TEST_KYLIN_FACT.LSTG_SITE_ID", 
-                         "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],
-                       
-                       [ "TEST_KYLIN_FACT.SELLER_ID", 
-                         "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-                         "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-                         "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
-                         "SELLER_COUNTRY.NAME" ],
-                         
-                       [ "TEST_KYLIN_FACT.ORDER_ID", 
-                         "TEST_ORDER.TEST_DATE_ENC", 
-                         "TEST_ORDER.TEST_TIME_ENC",
-                         "TEST_ORDER.BUYER_ID",
-                         "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
-                         "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
-                         "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
-                         "BUYER_COUNTRY.NAME" ] ]
+  "aggregation_groups": [
+    {
+      "includes": [
+        "TEST_KYLIN_FACT.CAL_DT",
+        "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+        "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+        "TEST_KYLIN_FACT.LSTG_SITE_ID",
+        "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+        "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+        "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+        "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+        "TEST_KYLIN_FACT.DEAL_YEAR"
+      ],
+      "select_rule": {
+        "hierarchy_dims": [
+          [
+            "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+            "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+            "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+            "TEST_KYLIN_FACT.LEAF_CATEG_ID"
+          ]
+        ],
+        "mandatory_dims": [],
+        "joint_dims": [
+          [
+            "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+            "TEST_KYLIN_FACT.LSTG_SITE_ID",
+            "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+            "TEST_KYLIN_FACT.DEAL_YEAR"
+          ]
+        ]
+      }
+    },
+    {
+      "includes": [
+        "TEST_KYLIN_FACT.CAL_DT",
+        "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+        "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+        "TEST_KYLIN_FACT.LSTG_SITE_ID",
+        "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+        "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+        "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+        "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+        "TEST_KYLIN_FACT.SELLER_ID",
+        "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+        "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+        "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+        "SELLER_COUNTRY.NAME",
+        "TEST_KYLIN_FACT.ORDER_ID",
+        "TEST_ORDER.TEST_DATE_ENC",
+        "TEST_ORDER.TEST_TIME_ENC",
+        "TEST_ORDER.BUYER_ID",
+        "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+        "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+        "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+        "BUYER_COUNTRY.NAME",
+        "SELLER_ACCOUNT.COUNTRY_ABBR",
+        "BUYER_ACCOUNT.COUNTRY_ABBR"
+      ],
+      "select_rule": {
+        "hierarchy_dims": [],
+        "mandatory_dims": [
+          "TEST_KYLIN_FACT.CAL_DT"
+        ],
+        "joint_dims": [
+          [
+            "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+            "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+            "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+            "TEST_KYLIN_FACT.LEAF_CATEG_ID"
+          ],
+          [
+            "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+            "TEST_KYLIN_FACT.LSTG_SITE_ID",
+            "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+            "SELLER_ACCOUNT.COUNTRY_ABBR",
+            "BUYER_ACCOUNT.COUNTRY_ABBR"
+          ],
+          [
+            "TEST_KYLIN_FACT.SELLER_ID",
+            "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+            "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+            "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+            "SELLER_COUNTRY.NAME"
+          ],
+          [
+            "TEST_KYLIN_FACT.ORDER_ID",
+            "TEST_ORDER.TEST_DATE_ENC",
+            "TEST_ORDER.TEST_TIME_ENC",
+            "TEST_ORDER.BUYER_ID",
+            "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+            "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+            "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+            "BUYER_COUNTRY.NAME"
+          ]
+        ]
+      }
     }
-  } ],
-  "notify_list" : null,
-  "status_need_notify" : [ ],
-  "auto_merge_time_ranges" : null,
-  "retention_range" : 0,
-  "engine_type" : 4,
-  "storage_type" : 2,
+  ],
+  "notify_list": null,
+  "status_need_notify": [],
+  "auto_merge_time_ranges": null,
+  "retention_range": 0,
+  "engine_type": 2,
+  "storage_type": 2,
   "override_kylin_properties": {
     "kylin.cube.algorithm": "LAYER"
   },


Mime
View raw message