kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [6/7] incubator-kylin git commit: whitelist
Date Mon, 30 Nov 2015 03:16:27 GMT
whitelist


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2d5f725b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2d5f725b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2d5f725b

Branch: refs/heads/KYLIN-242
Commit: 2d5f725bf3ea47972ed1fd4c2f67791124abba17
Parents: 2d5d3ef
Author: honma <honma@ebay.com>
Authored: Fri Nov 27 16:47:28 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Mon Nov 30 11:16:53 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/dataGen/FactTableGenerator.java   |   7 +-
 .../persistence/RootPersistentEntity.java       |   2 +
 .../org/apache/kylin/common/util/BasicTest.java |   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java    | 319 +++----
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  78 +-
 .../kylin/cube/cuboid/CuboidScheduler.java      | 235 +++--
 .../gridtable/CuboidToGridTableMapping.java     |  10 +-
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |   6 +-
 .../org/apache/kylin/cube/kv/RowConstants.java  |   2 +
 .../kylin/cube/model/AggregationGroup.java      | 261 ++++++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 179 ++--
 .../apache/kylin/cube/model/DimensionDesc.java  | 121 +--
 .../kylin/cube/model/HBaseMappingDesc.java      |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  73 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java | 227 +----
 .../org/apache/kylin/cube/model/SelectRule.java |  30 +
 .../kylin/cube/model/v2/CubeBuildTypeEnum.java  |  39 +
 .../apache/kylin/cube/model/v2/CubeDesc.java    | 867 +++++++++++++++++++
 .../kylin/cube/model/v2/DimensionDesc.java      | 239 +++++
 .../kylin/cube/model/v2/HBaseColumnDesc.java    | 138 +++
 .../cube/model/v2/HBaseColumnFamilyDesc.java    |  58 ++
 .../kylin/cube/model/v2/HBaseMappingDesc.java   |  96 ++
 .../kylin/cube/model/v2/HierarchyDesc.java      |  68 ++
 .../kylin/cube/model/v2/RowKeyColDesc.java      |  92 ++
 .../apache/kylin/cube/model/v2/RowKeyDesc.java  | 295 +++++++
 .../model/validation/CubeMetadataValidator.java |   9 +-
 .../cube/model/validation/ValidateContext.java  |   3 -
 .../rule/AggregationGroupOverlapRule.java       |  85 ++
 .../validation/rule/AggregationGroupRule.java   | 170 ++++
 .../rule/AggregationGroupSizeRule.java          |  66 --
 .../rule/IKylinValidationConstants.java         |   6 +-
 .../validation/rule/MandatoryColumnRule.java    |  75 --
 .../model/validation/rule/RowKeyAttrRule.java   |  16 +-
 .../cube/upgrade/V2/CubeDescUpgraderV2.java     | 290 +++++++
 .../cube/upgrade/V2/CubeMetadataUpgradeV2.java  | 177 ++++
 .../org/apache/kylin/cube/util/CubingUtils.java |  30 +-
 .../kylin/cube/AggregationGroupRuleTest.java    | 101 +++
 .../cube/AggregationGroupSizeRuleTest.java      | 101 ---
 .../kylin/cube/DictionaryManagerTest.java       |   4 +-
 .../kylin/cube/MandatoryColumnRuleTest.java     |  57 --
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  64 +-
 .../kylin/metadata/MetadataUpgradeTest.java     |   2 +-
 .../apache/kylin/dict/DictionaryManager.java    |  10 +-
 .../kylin/metadata/model/MeasureDesc.java       |  15 -
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |   2 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   3 +-
 .../mr/steps/MapContextGTRecordWriter.java      |   2 +-
 .../mr/steps/MergeCuboidFromStorageMapper.java  |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |   2 +-
 .../engine/mr/steps/MergeDictionaryStep.java    |   2 +-
 .../kylin/engine/mr/steps/CubeSamplingTest.java |   3 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   5 +-
 .../spark/cube/DefaultTupleConverter.java       |   3 +-
 .../localmeta/cube_desc/sample.json             | 257 ++++++
 .../apache/kylin/rest/service/CubeService.java  |   4 +-
 .../rest/controller/CubeControllerTest.java     |   2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v1/CubeTupleConverter.java       |   6 +-
 .../hbase/cube/v2/CubeTupleConverter.java       |   3 +-
 .../kylin/storage/hbase/steps/BulkLoadJob.java  |   2 +-
 .../storage/hbase/steps/CubeHFileMapper.java    |   2 +-
 .../storage/hbase/steps/CubeHTableUtil.java     |   2 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   2 +-
 .../storage/hbase/steps/HBaseMROutput2.java     |   4 +-
 .../hbase/steps/HBaseMROutput2Transition.java   |   2 +-
 .../hbase/steps/RowValueDecoderTest.java        |   4 +-
 68 files changed, 3773 insertions(+), 1274 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 5a0fee7..c033117 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -297,10 +297,9 @@ public class FactTableGenerator {
             JoinDesc jDesc = dim.getJoin();
             if (jDesc == null) {
                 // column on fact table used directly as a dimension
-                for (String aColumn : dim.getColumn()) {
-                    if (!factTableCol2LookupCol.containsKey(aColumn))
-                        usedCols.add(aColumn);
-                }
+                String aColumn = dim.getColumn();
+                if (!factTableCol2LookupCol.containsKey(aColumn))
+                    usedCols.add(aColumn);
             }
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index bc72c1e..c8177e6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -90,6 +90,8 @@ abstract public class RootPersistentEntity implements AclEntity, Serializable {
     public void setLastModified(long lastModified) {
         this.lastModified = lastModified;
     }
+    
+    
 
     public void updateRandomUuid() {
         setUuid(UUID.randomUUID().toString());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index d753a20..7c6a57f 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -88,6 +87,7 @@ public class BasicTest {
         }
     }
 
+    
     @Test
     public void testxx() throws InterruptedException {
         byte[][] data = new byte[10000000][];

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4592b15..4338f54 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -167,7 +167,7 @@ public class CubeManager implements IRealizationProvider {
             return null;
 
         DictionaryManager dictMgr = getDictionaryManager();
-        DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), "true", col, factTableValueProvider);
+        DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col, factTableValueProvider);
 
         if (dictInfo != null) {
             cubeSeg.putDictResPath(col, dictInfo.getResourcePath());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index d7e7d9c..b0774cf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -19,30 +19,38 @@
 package org.apache.kylin.cube.cuboid;
 
 import java.util.ArrayList;
-import java.util.BitSet;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
 import org.apache.kylin.metadata.model.TblColRef;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
+
 public class Cuboid implements Comparable<Cuboid> {
 
     private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
 
+    //smaller is better
+    public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
+        @Override
+        public int compare(Long o1, Long o2) {
+            return ComparisonChain.start().compare(Long.bitCount(o1), Long.bitCount(o2)).compare(o1, o2).result();
+        }
+    };
+
     public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
         return findById(cube, Bytes.toLong(cuboidID));
     }
@@ -64,29 +72,33 @@ public class Cuboid implements Comparable<Cuboid> {
     }
 
     public static boolean isValid(CubeDesc cube, long cuboidID) {
-        RowKeyDesc rowkey = cube.getRowkey();
-
-        if (cuboidID < 0) {
-            throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
+        for (AggregationGroup agg : cube.getAggregationGroups()) {
+            if (isValid(agg, cuboidID)) {
+                return true;
+            }
         }
 
-        if (checkBaseCuboid(rowkey, cuboidID)) {
-            return true;
-        }
+        return false;
+    }
 
-        if (checkMandatoryColumns(rowkey, cuboidID) == false) {
-            return false;
+    private static boolean isValid(AggregationGroup agg, long cuboidID) {
+        if (cuboidID < 0) {
+            throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
         }
-
-        if (checkAggregationGroup(rowkey, cuboidID) == false) {
-            return false;
+        if ((cuboidID & ~agg.getPartialCubeFullMask()) != 0) {
+            return false; //a cuboid's parent within agg is at most partialCubeFullMask
         }
+        return checkMandatoryColumns(agg, cuboidID) && checkHierarchy(agg, cuboidID);
+    }
 
-        if (checkHierarchy(rowkey, cuboidID) == false) {
-            return false;
+    public static List<AggregationGroup> getValidAggGroupForCuboid(CubeDesc cubeDesc, long cuboidID) {
+        List<AggregationGroup> ret = Lists.newArrayList();
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            if (isValid(agg, cuboidID)) {
+                ret.add(agg);
+            }
         }
-
-        return true;
+        return ret;
     }
 
     public static long getBaseCuboidId(CubeDesc cube) {
@@ -97,168 +109,96 @@ public class Cuboid implements Comparable<Cuboid> {
         return findById(cube, getBaseCuboidId(cube));
     }
 
-    private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
-        // add mandantory
-        RowKeyDesc rowkey = cubeDesc.getRowkey();
-        long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
-        if (cuboidID < mandatoryColumnMask) {
-            cuboidID = cuboidID | mandatoryColumnMask;
-        }
-
-        // add hierarchy
-        for (DimensionDesc dimension : cubeDesc.getDimensions()) {
-            HierarchyDesc[] hierarchies = dimension.getHierarchy();
-            boolean found = false;
-            long result = 0;
-            if (hierarchies != null && hierarchies.length > 0) {
-                for (int i = hierarchies.length - 1; i >= 0; i--) {
-                    TblColRef hColumn = hierarchies[i].getColumnRef();
-                    Integer index = rowkey.getColumnBitIndex(hColumn);
-                    long bit = 1L << index;
-
-                    if ((rowkey.getTailMask() & bit) > 0)
-                        continue; // ignore levels in tail, they don't participate
-
-                    if ((bit & cuboidID) > 0) {
-                        found = true;
-                    }
-
-                    if (found == true) {
-                        result = result | bit;
-                    }
-                }
-                cuboidID = cuboidID | result;
+    public static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
+        List<Long> candidates = Lists.newArrayList();
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            Long candidate = translateToValidCuboid(agg, cuboidID);
+            if (candidate != null) {
+                candidates.add(candidate);
             }
         }
 
-        // find the left-most aggregation group
-        long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
-        long leftover;
-        for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
-            if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) {
-                leftover = cuboidWithoutMandatory & ~mask.groupMask;
-
-                if (leftover == 0) {
-                    return cuboidID;
-                }
-
-                if (leftover != 0) {
-                    cuboidID = cuboidID | mask.leftoverMask;
-                    return cuboidID;
-                }
-            }
+        if (candidates.size() == 0) {
+            throw new IllegalStateException("Cannot find parent for :" + cuboidID);
         }
 
-        // doesn't have column in aggregation groups
-        leftover = cuboidWithoutMandatory & rowkey.getTailMask();
-        if (leftover == 0) {
-            // doesn't have column in tail group
-            if (cuboidWithoutMandatory != 0) {
-                return cuboidID;
-            } else {
-                // no column (except mandatory), add one column
-                long toAddCol = (1 << (BitSet.valueOf(new long[] { rowkey.getTailMask() }).cardinality()));
-                // check if the toAddCol belongs to any hierarchy
-                List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
-                if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
-                    for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
-                        long result = toAddCol & hierarchyMasks.fullMask;
-                        if (result > 0) {
-                            // replace it with the root col in hierarchy
-                            toAddCol = hierarchyMasks.allMasks[0];
-                            break;
-                        }
-                    }
-                }
-                cuboidID = cuboidID | toAddCol;
-                return cuboidID;
-            }
-        }
-
-        // has column in tail group
-        cuboidID = cuboidID | rowkey.getTailMask();
-        return cuboidID;
-
+        return Collections.min(candidates, cuboidSelectComparator);
     }
 
-    /** Breadth-First-Search
-     * @deprecated due to poor performance
-     * @param cube
-     * @param cuboidID
-     * @return
-     */
-    private static long translateToValidCuboidDeprecated(CubeDesc cube, long cuboidID) {
-        if (Cuboid.isValid(cube, cuboidID)) {
-            return cuboidID;
+    private static Long translateToValidCuboid(AggregationGroup agg, long cuboidID) {
+        if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+            //the partial cube might not contain all required dims
+            return null;
         }
 
-        HashSet<Long> dedupped = new HashSet<Long>();
-        Queue<Long> queue = new LinkedList<Long>();
-        List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID);
-
-        // check each parent
-        addToQueue(queue, parents, dedupped);
-        while (queue.size() > 0) {
-            long parent = pollFromQueue(queue, dedupped);
-            if (Cuboid.isValid(cube, parent)) {
-                return parent;
-            } else {
-                addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped);
+        // add mandantory
+        cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+        // add hierarchy
+        for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+            long fullMask = hierarchyMask.fullMask;
+            long intersect = cuboidID & fullMask;
+            if (intersect != 0 && intersect != fullMask) {
+                long lsb = Long.lowestOneBit(intersect);
+                long fillMask = fullMask & ~(lsb - 1);
+                cuboidID |= fillMask;
             }
         }
-        return -1;
-    }
-
-    private static List<Long> getAllPossibleParents(CubeDesc cube, long cuboidID) {
-        List<Long> allPossibleParents = new ArrayList<Long>();
 
-        for (int i = 0; i < cube.getRowkey().getRowKeyColumns().length; i++) {
-            long mask = 1L << i;
-            long parentId = cuboidID | mask;
-            if (parentId != cuboidID) {
-                allPossibleParents.add(parentId);
+        // add joint dims
+        for (Long joint : agg.getJointDims()) {
+            if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+                cuboidID = cuboidID | joint;
             }
         }
 
-        return allPossibleParents;
-    }
-
-    private static void addToQueue(Queue<Long> queue, List<Long> parents, HashSet<Long> dedupped) {
-        Collections.sort(parents);
-        for (Long p : parents) {
-            if (!dedupped.contains(p)) {
-                dedupped.add(p);
-                queue.offer(p);
+        if (cuboidID != 0) {
+            return cuboidID;
+        } else {
+            // no column, add one column
+            long nonJointDims = removeBits(agg.getPartialCubeFullMask(), agg.getJointDims());
+            if (nonJointDims != 0) {
+                long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+                    @Override
+                    public Long apply(HierarchyMask input) {
+                        return input.fullMask;
+                    }
+                }));
+                if (nonJointNonHierarchy != 0) {
+                    //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+                    return Long.lowestOneBit(nonJointNonHierarchy);
+                } else {
+                    //choose from a hierarchy that does not intersect with any joint dim, only check level 1 
+                    long allJointDims = agg.getJointDimsMask();
+                    int index = 0;
+                    for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+                        long dim = hierarchyMask.allMasks[index];
+                        if ((nonJointDims & allJointDims) == 0) {
+                            return dim;
+                        }
+                    }
+                }
             }
-        }
-    }
 
-    private static long pollFromQueue(Queue<Long> queue, HashSet<Long> dedupped) {
-        long element = queue.poll();
-        dedupped.remove(element);
-        return element;
+            return Collections.min(agg.getJointDims(), cuboidSelectComparator);
+        }
     }
 
-    private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
-        long baseCuboidId = rowkey.getFullMask();
-        if (cuboidID > baseCuboidId) {
-            throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
+    private static long removeBits(long original, Collection<Long> toRemove) {
+        long ret = original;
+        for (Long joint : toRemove) {
+            ret = ret & ~joint;
         }
-        return baseCuboidId == cuboidID;
+        return ret;
     }
 
-    private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
-        long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
-
-        // note the all-zero cuboid (except for mandatory) is not valid
-        if (cuboidID <= mandatoryColumnMask)
-            return false;
-
+    private static boolean checkMandatoryColumns(AggregationGroup agg, long cuboidID) {
+        long mandatoryColumnMask = agg.getMandatoryColumnMask();
         return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
     }
 
-    private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
-        List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
+    private static boolean checkHierarchy(AggregationGroup agg, long cuboidID) {
+        List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
         // if no hierarchy defined in metadata
         if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
             return true;
@@ -279,23 +219,9 @@ public class Cuboid implements Comparable<Cuboid> {
         return true;
     }
 
-    private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
-        long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
-        long leftover;
-        for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
-            if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
-                leftover = cuboidWithoutMandatory & ~mask.groupMask;
-                return leftover == 0 || leftover == mask.leftoverMask;
-            }
-        }
-
-        leftover = cuboidWithoutMandatory & rowkey.getTailMask();
-        return leftover == 0 || leftover == rowkey.getTailMask();
-    }
-
     // ============================================================================
 
-    private CubeDesc cube;
+    private CubeDesc cubeDesc;
     private final long inputID;
     private final long id;
     private final byte[] idBytes;
@@ -305,8 +231,8 @@ public class Cuboid implements Comparable<Cuboid> {
     private volatile CuboidToGridTableMapping cuboidToGridTableMapping = null;
 
     // will translate the cuboidID if it is not valid
-    private Cuboid(CubeDesc cube, long originalID, long validID) {
-        this.cube = cube;
+    private Cuboid(CubeDesc cubeDesc, long originalID, long validID) {
+        this.cubeDesc = cubeDesc;
         this.inputID = originalID;
         this.id = validID;
         this.idBytes = Bytes.toBytes(id);
@@ -316,7 +242,7 @@ public class Cuboid implements Comparable<Cuboid> {
 
     private List<TblColRef> translateIdToColumns(long cuboidID) {
         List<TblColRef> dimesnions = new ArrayList<TblColRef>();
-        RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
+        RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
         for (int i = 0; i < allColumns.length; i++) {
             // NOTE: the order of column in list!!!
             long bitmask = 1L << allColumns[i].getBitIndex();
@@ -335,23 +261,30 @@ public class Cuboid implements Comparable<Cuboid> {
 
     // higher level in hierarchy can be ignored when counting aggregation columns
     private long eliminateHierarchyAggregation(long id) {
-        List<HierarchyMask> hierarchyMaskList = cube.getRowkey().getHierarchyMasks();
-        if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
-            for (HierarchyMask hierMask : hierarchyMaskList) {
-                long[] allMasks = hierMask.allMasks;
-                for (int i = allMasks.length - 1; i > 0; i--) {
-                    long bit = allMasks[i] ^ allMasks[i - 1];
-                    if ((inputID & bit) != 0) {
-                        id &= ~allMasks[i - 1];
+        long finalId = id;
+
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            long temp = id;
+            List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
+            if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
+                for (HierarchyMask hierMask : hierarchyMaskList) {
+                    long[] allMasks = hierMask.allMasks;
+                    for (int i = allMasks.length - 1; i > 0; i--) {
+                        long bit = allMasks[i] ^ allMasks[i - 1];
+                        if ((inputID & bit) != 0) {
+                            temp &= ~allMasks[i - 1];
+                            if (temp < finalId)
+                                finalId = temp;
+                        }
                     }
                 }
             }
         }
-        return id;
+        return finalId;
     }
 
-    public CubeDesc getCube() {
-        return cube;
+    public CubeDesc getCubeDesc() {
+        return cubeDesc;
     }
 
     public List<TblColRef> getColumns() {
@@ -375,10 +308,6 @@ public class Cuboid implements Comparable<Cuboid> {
         return inputID;
     }
 
-    public boolean useAncestor() {
-        return inputID != id;
-    }
-
     public boolean requirePostAggregation() {
         return requirePostAggregation;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 7a4b00a..777d809 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -26,9 +26,6 @@ import java.util.TreeSet;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
 
 /**
  * @author yangli9
@@ -41,11 +38,11 @@ public class CuboidCLI {
 
         if ("test".equals(args[0])) {
             CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
-            simulateCuboidGeneration(cubeDesc);
+            simulateCuboidGeneration(cubeDesc, true);
         }
     }
 
-    public static int simulateCuboidGeneration(CubeDesc cube) {
+    public static int simulateCuboidGeneration(CubeDesc cube, boolean validate) {
         CuboidScheduler scheduler = new CuboidScheduler(cube);
 
         long baseCuboid = Cuboid.getBaseCuboidId(cube);
@@ -64,20 +61,16 @@ public class CuboidCLI {
                 cuboidQueue.push(sc);
             }
         }
-        
-        /** disable this due to poor performance when dimension number is big
-        TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
-        if (enumCuboids.equals(cuboidSet) == false) {
-            throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
-        }
-         */
 
-        int mathCount = mathCalcCuboidCount(cube);
-        if (mathCount != cuboidSet.size()) {
-            throw new IllegalStateException("Math cuboid count " + mathCount + ", but actual cuboid count " + cuboidSet.size());
+        if (validate) {
+            //only run this for test purpose, performance is bad when # of dims is large
+            TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
+            if (enumCuboids.equals(cuboidSet) == false) {
+                throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
+            }
         }
 
-        return mathCount;
+        return cuboidSet.size();
 
     }
 
@@ -93,7 +86,7 @@ public class CuboidCLI {
     }
 
     public static int[] calculateAllLevelCount(CubeDesc cube) {
-        int levels = cube.getRowkey().getNCuboidBuildLevels();
+        int levels = cube.getBuildLevel();
         int[] allLevelCounts = new int[levels + 1];
 
         CuboidScheduler scheduler = new CuboidScheduler(cube);
@@ -116,55 +109,4 @@ public class CuboidCLI {
         return allLevelCounts;
     }
 
-    public static int mathCalcCuboidCount(CubeDesc cube) {
-        int result = 1; // 1 for base cuboid
-
-        RowKeyDesc rowkey = cube.getRowkey();
-        AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
-        for (int i = 0; i < aggrGroupMasks.length; i++) {
-            boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
-            result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
-        }
-
-        return result;
-    }
-
-    private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
-        long groupMask = aggrGroupMask.groupMask;
-        int n = mathCalcCuboidCount_combination(rowkey, groupMask);
-        n -= 2; // exclude group all 1 and all 0
-
-        long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
-        if (nonUniqueMask > 0) {
-            // exclude duplicates caused by non-unique columns
-            // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
-            n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
-        }
-
-        if (hasTail) {
-            n *= 2; // tail being 1 and 0
-            n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
-        }
-
-        return n;
-    }
-
-    private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
-        if (colMask == 0) // no column selected
-            return 0;
-
-        int count = 1;
-
-        for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
-            long hierBits = colMask & hierMask.fullMask;
-            if (hierBits != 0) {
-                count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
-                colMask &= ~hierBits;
-            }
-        }
-
-        count *= Math.pow(2, Long.bitCount(colMask));
-
-        return count;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 7596601..93cab06 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -21,180 +21,171 @@ package org.apache.kylin.cube.cuboid;
 /** 
  */
 
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 public class CuboidScheduler {
 
-    private final CubeDesc cubeDef;
-    private final int size;
+    private final CubeDesc cubeDesc;
     private final long max;
     private final Map<Long, List<Long>> cache;
 
-    public CuboidScheduler(CubeDesc cube) {
-        this.cubeDef = cube;
-        this.size = cube.getRowkey().getRowKeyColumns().length;
+    public CuboidScheduler(CubeDesc cubeDesc) {
+        this.cubeDesc = cubeDesc;
+        int size = this.cubeDesc.getRowkey().getRowKeyColumns().length;
         this.max = (long) Math.pow(2, size) - 1;
         this.cache = new ConcurrentHashMap<Long, List<Long>>();
     }
 
-    public int getCuboidCount() {
-        return getCuboidCount(Cuboid.getBaseCuboidId(cubeDef));
-    }
-
-    private int getCuboidCount(long cuboidId) {
-        int r = 1;
-        for (Long child : getSpanningCuboid(cuboidId)) {
-            r += getCuboidCount(child);
+    public long getParent(long child) {
+        List<Long> candidates = Lists.newArrayList();
+        long baseCuboidID = Cuboid.getBaseCuboidId(cubeDesc);
+        if (child == baseCuboidID) {
+            throw new IllegalStateException();
         }
-        return r;
-    }
-
-    public List<Long> getSpanningCuboid(long cuboid) {
-        if (cuboid > max || cuboid < 0) {
-            throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
-        }
-
-        List<Long> result = cache.get(cuboid);
-        if (result != null) {
-            return result;
-        }
-
-        // smaller sibling's children
-        Collection<Long> allPrevOffspring = new HashSet<Long>();
-        for (Long sibling : findSmallerSibling(cuboid)) {
-            Collection<Long> prevOffsprings = generateChildren(sibling);
-            allPrevOffspring.addAll(prevOffsprings);
-        }
-
-        // my children is my generation excluding smaller sibling's generation
-        result = new ArrayList<Long>();
-        for (Long offspring : generateChildren(cuboid)) {
-            if (!allPrevOffspring.contains(offspring)) {
-                result.add(offspring);
+        for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, child)) {
+            boolean thisAggContributed = false;
+            if (agg.getPartialCubeFullMask() == child) {
+                candidates.add(baseCuboidID);
+                continue;
             }
-        }
 
-        cache.put(cuboid, result);
-        return result;
-    }
+            //+1 dim
 
-    private Collection<Long> generateChildren(long cuboid) {
-        Collection<Long> result = new HashSet<Long>();
+            //add one normal dim (only try the lowest dim)
+            long normalDimsMask = agg.getNormalDimsMask();
+            if (normalDimsMask != 0) {
+                candidates.add(child | Long.lowestOneBit(normalDimsMask));
+                thisAggContributed = true;
+            }
 
-        // generate zero tail cuboid -- the one with all 1 in the first
-        // aggregation group and all 0 for the rest bits
-        generateZeroTailBase(cuboid, result);
+            for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+                for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+                    if ((child & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+                        if (i == hierarchyMask.allMasks.length - 1) {
+                            continue;//match the full hierarchy
+                        }
+                        if ((agg.getJointDimsMask() & hierarchyMask.dims[i + 1]) == 0) {
+                            //only when the hierarchy dim is not among joints
+                            candidates.add(child | hierarchyMask.dims[i + 1]);
+                            thisAggContributed = true;
+                        }
+                        break;//if hierarchyMask 111 is matched, won't check 110 or 100
+                    }
+                }
+            }
 
-        RowKeyDesc rowkey = cubeDef.getRowkey();
-        long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
-        for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
-            if (belongTo(cuboidWithoutMandatory, mask) == false)
+            if (thisAggContributed) {
+                //next section is going to append more than 2 dim to child
+                //thisAggContributed means there's already 1 dim added to child
+                //which can safely prune the 2+ dim candidates.
                 continue;
+            }
 
-            long[] groupOneBitMasks = mask.groupOneBitMasks;
-            for (int i = 0; i < groupOneBitMasks.length; i++) {
-                long oneBit = groupOneBitMasks[i];
-                if ((cuboid & oneBit) == 0)
-                    continue;
-
-                long child = cuboid ^ oneBit;
-                if (Cuboid.isValid(cubeDef, child)) {
-                    result.add(child);
+            //2+ dim candidates
+            for (long joint : agg.getJointDims()) {
+                if ((child & joint) == 0) {
+                    candidates.add(child | joint);
                 }
             }
-
-            if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
-                break;
         }
 
-        return result;
+        return Collections.min(candidates, Cuboid.cuboidSelectComparator);
     }
 
-    private void generateZeroTailBase(long cuboid, Collection<Long> result) {
-        RowKeyDesc rowkey = cubeDef.getRowkey();
-
-        long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
-
-        for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
-            if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
-                long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
-                if (zeroTail > 0 && zeroTail != cuboid) {
-                    result.add(zeroTail);
+    public Set<Long> getPotentialChildren(long parent) {
+        HashSet<Long> set = Sets.newHashSet();
+        if (parent == Cuboid.getBaseCuboidId(cubeDesc)) {
+            //base cuboid is responsible for spawning each agg group's root
+            for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+                long partialCubeFullMask = agg.getPartialCubeFullMask();
+                if (partialCubeFullMask != parent) {
+                    set.add(partialCubeFullMask);
                 }
             }
-            if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
-                break;
+            return set;
         }
-    }
 
-    public Collection<Long> findSmallerSibling(long cuboid) {
-        if (!Cuboid.isValid(cubeDef, cuboid)) {
-            return Collections.emptyList();
+        if (Long.bitCount(parent) == 1) {
+            //do not aggregate apex cuboid
+            return set;
         }
 
-        RowKeyDesc rowkey = cubeDef.getRowkey();
+        for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, parent)) {
 
-        // do combination in all related groups
-        long groupAllBitMask = 0;
-        for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
-            if ((mask.groupMask & cuboid) > 0) {
-                groupAllBitMask |= mask.groupMask;
+            //normal dim section
+            for (long normalDimMask : agg.getNormalDims()) {
+                if ((parent & normalDimMask) != 0) {
+                    set.add(parent ^ normalDimMask);
+                }
             }
+
+            for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+                for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+                    if ((parent & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+                        if ((agg.getJointDimsMask() & hierarchyMask.dims[i]) == 0) {
+                            //only when the hierarchy dim is not among joints
+                            set.add(parent ^ hierarchyMask.dims[i]);
+                        }
+                        break;//if hierarchyMask 111 is matched, won't check 110 or 100
+                    }
+                }
+            }
+
+            //joint dim section
+            for (long joint : agg.getJointDims()) {
+                if ((parent & joint) == joint) {
+                    set.add(parent ^ joint);
+                }
+            }
+
         }
 
-        long groupBitValue = cuboid & groupAllBitMask;
-        long leftBitValue = cuboid & ~groupAllBitMask;
-        long[] groupOneBits = bits(groupAllBitMask);
+        return set;
+    }
 
-        Collection<Long> siblings = new HashSet<Long>();
-        combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
-        return siblings;
+    public int getCuboidCount() {
+        return getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
     }
 
-    private long[] bits(long groupAllBitMask) {
-        int size = Long.bitCount(groupAllBitMask);
-        long[] r = new long[size];
-        long l = groupAllBitMask;
-        int i = 0;
-        while (l != 0) {
-            long bit = Long.highestOneBit(l);
-            r[i++] = bit;
-            l ^= bit;
+    private int getCuboidCount(long cuboidId) {
+        int r = 1;
+        for (Long child : getSpanningCuboid(cuboidId)) {
+            r += getCuboidCount(child);
         }
         return r;
     }
 
-    private void combination(long cuboid, Collection<Long> siblings, long[] bitMasks, int offset, long bitValue, int k) {
-        if (k == 0) {
-            if (Cuboid.isValid(cubeDef, bitValue)) {
-                siblings.add(bitValue);
-            }
-        } else {
-            for (int i = offset; i < bitMasks.length; i++) {
-                long newBitValue = bitValue | bitMasks[i];
-                if (newBitValue < cuboid) {
-                    combination(cuboid, siblings, bitMasks, i + 1, newBitValue, k - 1);
-                }
+    public List<Long> getSpanningCuboid(long cuboid) {
+        if (cuboid > max || cuboid < 0) {
+            throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
+        }
+
+        List<Long> result = cache.get(cuboid);
+        if (result != null) {
+            return result;
+        }
+
+        result = Lists.newArrayList();
+        Set<Long> potentials = getPotentialChildren(cuboid);
+        for (Long potential : potentials) {
+            if (getParent(potential) == cuboid) {
+                result.add(potential);
             }
         }
-    }
 
-    private boolean belongTo(long cuboidWithoutMandatory, AggrGroupMask mask) {
-        long groupBits = cuboidWithoutMandatory & mask.groupMask;
-        long leftoverBits = cuboidWithoutMandatory & mask.leftoverMask;
-        return groupBits > 0 && (leftoverBits == 0 || leftoverBits == mask.leftoverMask);
+        cache.put(cuboid, result);
+        return result;
     }
 
     public int getCardinality(long cuboid) {
@@ -206,7 +197,7 @@ public class CuboidScheduler {
     }
 
     public List<Long> getAllCuboidIds() {
-        final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDef);
+        final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         List<Long> result = Lists.newArrayList();
         getSubCuboidIds(baseCuboidId, result);
         return result;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index c95e932..428137a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -61,7 +61,7 @@ public class CuboidToGridTableMapping {
 
         // column blocks of metrics
         ArrayList<BitSet> metricsColBlocks = Lists.newArrayList();
-        for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+        for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
             for (int i = 0; i < familyDesc.getColumns().length; i++) {
                 metricsColBlocks.add(new BitSet());
             }
@@ -69,7 +69,7 @@ public class CuboidToGridTableMapping {
         
         // metrics
         metrics2gt = LinkedListMultimap.create();
-        for (MeasureDesc measure :cuboid.getCube().getMeasures()) {
+        for (MeasureDesc measure :cuboid.getCubeDesc().getMeasures()) {
             // Count distinct & holistic count distinct are equals() but different.
             // Ensure the holistic version if exists is always the first.
             FunctionDesc func = measure.getFunction();
@@ -85,7 +85,7 @@ public class CuboidToGridTableMapping {
             
             // map to column block
             int cbIdx = 0;
-            for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+            for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
                 for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
                     if (hbaseColDesc.containsMeasure(measure.getName())) {
                         metricsColBlocks.get(cbIdx).set(gtColIdx);
@@ -102,7 +102,7 @@ public class CuboidToGridTableMapping {
         }
         
         nMetrics = gtColIdx - nDimensions;
-        assert nMetrics == cuboid.getCube().getMeasures().size();
+        assert nMetrics == cuboid.getCubeDesc().getMeasures().size();
     }
 
     public int getColumnCount() {
@@ -156,7 +156,7 @@ public class CuboidToGridTableMapping {
 
     public Map<Integer, Integer> getDependentMetricsMap() {
         Map<Integer, Integer> result = Maps.newHashMap();
-        List<MeasureDesc> measures = cuboid.getCube().getMeasures();
+        List<MeasureDesc> measures = cuboid.getCubeDesc().getMeasures();
         for (MeasureDesc child : measures) {
             if (child.getDependentMeasureRef() != null) {
                 boolean ok = false;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index f0ee372..2fbd45a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -54,11 +54,11 @@ public final class InMemCubeBuilderUtils {
     }
 
     public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
-        BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
+        int cardinality = Long.bitCount(cuboidId);
         BitSet dimension = new BitSet();
-        dimension.set(0, bitSet.cardinality());
+        dimension.set(0, cardinality);
         BitSet metrics = new BitSet();
-        metrics.set(bitSet.cardinality(), bitSet.cardinality() + measureCount);
+        metrics.set(cardinality, cardinality + measureCount);
         return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
index 62dea02..3510915 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.kv;
 
 public class RowConstants {
 
+    public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
+
     // row key fixed length place holder
     public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
     // row key lower bound

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
new file mode 100644
index 0000000..2e57583
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class AggregationGroup {
+    public static class HierarchyMask {
+        public long fullMask;// 00000111
+        public long[] allMasks;// 00000100,00000110,00000111
+        public long[] dims;// 00000100,00000100,00000001
+    }
+
+    @JsonProperty("includes")
+    private String[] includes;
+    @JsonProperty("select_rule")
+    private SelectRule selectRule;
+
+    //computed
+    private long partialCubeFullMask;
+    private long mandatoryColumnMask;
+    private List<HierarchyMask> hierarchyMasks;
+    private List<Long> jointDims;//each long is a group
+    private long jointDimsMask;
+    private long normalDimsMask;
+    private long hierarchyDimsMask;
+    private List<Long> normalDims;//each long is a single dim
+
+    public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) {
+        Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation();
+
+        buildPartialCubeFullMask(colNameAbbr, rowKeyDesc);
+        buildMandatoryColumnMask(colNameAbbr, rowKeyDesc);
+        buildHierarchyMasks(colNameAbbr, rowKeyDesc);
+        buildJointColumnMask(colNameAbbr, rowKeyDesc);
+        buildJointDimsMask();
+        buildNormalDimsMask();
+        buildHierarchyDimsMask();
+    }
+
+    private void buildPartialCubeFullMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+        Preconditions.checkState(this.includes != null);
+        Preconditions.checkState(this.includes.length != 0);
+
+        partialCubeFullMask = 0L;
+        for (String dim : this.includes) {
+            TblColRef hColumn = colNameAbbr.get(dim);
+            Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+            long bit = 1L << index;
+            partialCubeFullMask |= bit;
+        }
+    }
+
+    private void buildJointColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+        jointDims = Lists.newArrayList();
+
+        for (String[] joint_dims : this.selectRule.joint_dims) {
+            if (joint_dims == null || joint_dims.length == 0) {
+                continue;
+            }
+
+            long joint = 0L;
+            for (int i = 0; i < joint_dims.length; i++) {
+                TblColRef hColumn = colNameAbbr.get(joint_dims[i]);
+                Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+                long bit = 1L << index;
+                joint |= bit;
+            }
+
+            Preconditions.checkState(joint != 0);
+            jointDims.add(joint);
+        }
+    }
+
+    private void buildMandatoryColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+        mandatoryColumnMask = 0L;
+
+        String[] mandatory_dims = this.selectRule.mandatory_dims;
+        if (mandatory_dims == null || mandatory_dims.length == 0) {
+            return;
+        }
+
+        for (String dim : mandatory_dims) {
+            TblColRef hColumn = colNameAbbr.get(dim);
+            Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+            mandatoryColumnMask |= 1 << index;
+        }
+
+    }
+
+    private void buildHierarchyMasks(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+
+        HierarchyMask mask = new HierarchyMask();
+        this.hierarchyMasks = new ArrayList<HierarchyMask>();
+
+        for (String[] hierarchy_dims : this.selectRule.hierarchy_dims) {
+            if (hierarchy_dims == null || hierarchy_dims.length == 0) {
+                continue;
+            }
+
+            ArrayList<Long> allMaskList = new ArrayList<Long>();
+            ArrayList<Long> dimList = new ArrayList<Long>();
+            for (int i = 0; i < hierarchy_dims.length; i++) {
+                TblColRef hColumn = colNameAbbr.get(hierarchy_dims[i]);
+                Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+                long bit = 1L << index;
+
+                //                if ((tailMask & bit) > 0)
+                //                    continue; // ignore levels in tail, they don't participate
+                //                // aggregation group combination anyway
+
+                mask.fullMask |= bit;
+                allMaskList.add(mask.fullMask);
+                dimList.add(bit);
+            }
+
+            Preconditions.checkState(allMaskList.size() == dimList.size());
+            mask.allMasks = new long[allMaskList.size()];
+            mask.dims = new long[dimList.size()];
+            for (int i = 0; i < allMaskList.size(); i++) {
+                mask.allMasks[i] = allMaskList.get(i);
+                mask.dims[i] = dimList.get(i);
+            }
+
+            this.hierarchyMasks.add(mask);
+
+        }
+
+    }
+
+    private void buildNormalDimsMask() {
+        //no joint, no hierarchy, no mandatory
+        long leftover = partialCubeFullMask & ~mandatoryColumnMask;
+        leftover &= ~this.jointDimsMask;
+        for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+            leftover &= ~hierarchyMask.fullMask;
+        }
+
+        this.normalDimsMask = leftover;
+        this.normalDims = bits(leftover);
+    }
+
+    private void buildHierarchyDimsMask() {
+        long ret = 0;
+        for (HierarchyMask mask : hierarchyMasks) {
+            ret |= mask.fullMask;
+        }
+        this.hierarchyDimsMask = ret;
+    }
+
+    private List<Long> bits(long x) {
+        List<Long> r = Lists.newArrayList();
+        long l = x;
+        while (l != 0) {
+            long bit = Long.lowestOneBit(l);
+            r.add(bit);
+            l ^= bit;
+        }
+        return r;
+    }
+
+    public void buildJointDimsMask() {
+        long ret = 0;
+        for (long x : jointDims) {
+            ret |= x;
+        }
+        this.jointDimsMask = ret;
+    }
+
+    public long getMandatoryColumnMask() {
+        return mandatoryColumnMask;
+    }
+
+    public List<HierarchyMask> getHierarchyMasks() {
+        return hierarchyMasks;
+    }
+
+    public int getBuildLevel() {
+        int ret = 0;
+        ret += getNormalDims().size();
+        long allHierarchyMask = 0L;
+        for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+            ret += hierarchyMask.allMasks.length;
+            allHierarchyMask |= hierarchyMask.fullMask;
+        }
+        for (Long joint : jointDims) {
+            if ((joint & allHierarchyMask) == 0) {
+                ret += 1;
+            }
+        }
+        return ret - 1;
+    }
+
+    public void setIncludes(String[] includes) {
+        this.includes = includes;
+    }
+
+    public void setSelectRule(SelectRule selectRule) {
+        this.selectRule = selectRule;
+    }
+
+    public List<Long> getJointDims() {
+        return jointDims;
+    }
+
+    public long getJointDimsMask() {
+        return jointDimsMask;
+    }
+
+    public long getNormalDimsMask() {
+        return normalDimsMask;
+    }
+
+    public long getHierarchyDimsMask() {
+        return hierarchyDimsMask;
+    }
+
+    public List<Long> getNormalDims() {
+        return normalDims;
+    }
+
+    public long getPartialCubeFullMask() {
+        return partialCubeFullMask;
+    }
+
+    public String[] getIncludes() {
+        return includes;
+    }
+
+    public SelectRule getSelectRule() {
+        return selectRule;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2250945..2df9635 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -32,6 +31,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
@@ -58,6 +59,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -67,7 +70,7 @@ import com.google.common.collect.Maps;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
 public class CubeDesc extends RootPersistentEntity {
 
-    public static enum DeriveType {
+    public enum DeriveType {
         LOOKUP, PK_FK
     }
 
@@ -110,6 +113,8 @@ public class CubeDesc extends RootPersistentEntity {
     private RowKeyDesc rowkey;
     @JsonProperty("hbase_mapping")
     private HBaseMappingDesc hbaseMapping;
+    @JsonProperty("aggregation_gropus")
+    private List<AggregationGroup> aggregationGroups;
     @JsonProperty("signature")
     private String signature;
     @JsonProperty("notify_list")
@@ -171,16 +176,14 @@ public class CubeDesc extends RootPersistentEntity {
     }
 
     /**
-     * Find FunctionDesc by Full Expression.
-     *
-     * @return
+     * @return all functions from each measure.
      */
-    public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
+    public List<FunctionDesc> listAllFunctions() {
+        List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
         for (MeasureDesc m : measures) {
-            if (m.getFunction().equals(manualFunc))
-                return m.getFunction();
+            functions.add(m.getFunction());
         }
-        return null;
+        return functions;
     }
 
     public TblColRef findColumnRef(String table, String column) {
@@ -191,14 +194,6 @@ public class CubeDesc extends RootPersistentEntity {
             return cols.get(column);
     }
 
-    public DimensionDesc findDimensionByColumn(TblColRef col) {
-        for (DimensionDesc dim : dimensions) {
-            if (ArrayUtils.contains(dim.getColumnRefs(), col))
-                return dim;
-        }
-        return null;
-    }
-
     public DimensionDesc findDimensionByTable(String lookupTableName) {
         lookupTableName = lookupTableName.toUpperCase();
         for (DimensionDesc dim : dimensions)
@@ -207,28 +202,6 @@ public class CubeDesc extends RootPersistentEntity {
         return null;
     }
 
-    public DimensionDesc findDimensionByName(String dimName) {
-        dimName = dimName.toUpperCase();
-        for (DimensionDesc dim : dimensions) {
-            if (dimName.equals(dim.getName()))
-                return dim;
-        }
-        return null;
-    }
-
-    /**
-     * Get all functions from each measure.
-     *
-     * @return
-     */
-    public List<FunctionDesc> listAllFunctions() {
-        List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
-        for (MeasureDesc m : measures) {
-            functions.add(m.getFunction());
-        }
-        return functions;
-    }
-
     public boolean isDerived(TblColRef col) {
         return derivedToHostMap.containsKey(col);
     }
@@ -270,14 +243,6 @@ public class CubeDesc extends RootPersistentEntity {
 
     // ============================================================================
 
-    public HBaseMappingDesc getHBaseMapping() {
-        return hbaseMapping;
-    }
-
-    public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
-        this.hbaseMapping = hbaseMapping;
-    }
-
     public KylinConfig getConfig() {
         return config;
     }
@@ -354,6 +319,14 @@ public class CubeDesc extends RootPersistentEntity {
         this.rowkey = rowkey;
     }
 
+    public List<AggregationGroup> getAggregationGroups() {
+        return aggregationGroups;
+    }
+
+    public void setAggregationGroups(List<AggregationGroup> aggregationGroups) {
+        this.aggregationGroups = aggregationGroups;
+    }
+
     public String getSignature() {
         return signature;
     }
@@ -389,12 +362,23 @@ public class CubeDesc extends RootPersistentEntity {
 
         if (!name.equals(cubeDesc.name))
             return false;
+
         if (!getFactTable().equals(cubeDesc.getFactTable()))
             return false;
 
         return true;
     }
 
+    public int getBuildLevel() {
+        return Collections.max(Collections2.transform(aggregationGroups, new Function<AggregationGroup, Integer>() {
+            @Nullable
+            @Override
+            public Integer apply(AggregationGroup input) {
+                return input.getBuildLevel();
+            }
+        })) + 1;
+    }
+
     @Override
     public int hashCode() {
         int result = 0;
@@ -414,13 +398,19 @@ public class CubeDesc extends RootPersistentEntity {
         }
         return calculateSignature().equals(getSignature());
     }
-    
+
     public String calculateSignature() {
-        MessageDigest md = null;
+        MessageDigest md;
         try {
             md = MessageDigest.getInstance("MD5");
             StringBuilder sigString = new StringBuilder();
-            sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
+            sigString.append(this.name).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.modelName)).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.dimensions)).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.measures)).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.rowkey)).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.aggregationGroups)).append("|")//
+                    .append(JsonUtil.writeValueAsString(this.hbaseMapping));
 
             byte[] signature = md.digest(sigString.toString().getBytes());
             return new String(Base64.encodeBase64(signature));
@@ -455,11 +445,14 @@ public class CubeDesc extends RootPersistentEntity {
             dim.init(this, tables);
         }
 
-        sortDimAndMeasure();
         initDimensionColumns();
         initMeasureColumns();
 
         rowkey.init(this);
+        for (AggregationGroup agg : this.aggregationGroups) {
+            agg.init(this, rowkey);
+        }
+
         if (hbaseMapping != null) {
             hbaseMapping.init(this);
         }
@@ -479,35 +472,32 @@ public class CubeDesc extends RootPersistentEntity {
 
             // init dimension columns
             ArrayList<TblColRef> dimCols = Lists.newArrayList();
-            String[] colStrs = dim.getColumn();
+            String colStrs = dim.getColumn();
 
             // when column is omitted, special case
-            if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
+            if ((colStrs == null && dim.isDerived()) || ("{FK}".equalsIgnoreCase(colStrs))) {
                 for (TblColRef col : join.getForeignKeyColumns()) {
                     dimCols.add(initDimensionColRef(col));
                 }
             }
             // normal case
             else {
-                if (colStrs == null || colStrs.length == 0)
+                if (StringUtils.isEmpty(colStrs))
                     throw new IllegalStateException("Dimension column must not be blank " + dim);
 
-                for (String colStr : colStrs) {
-                    dimCols.add(initDimensionColRef(dim, colStr));
-                }
+                dimCols.add(initDimensionColRef(dim, colStrs));
 
-                // fill back column ref in hierarchy
-                if (dim.isHierarchy()) {
-                    for (int i = 0; i < dimCols.size(); i++)
-                        dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
-                }
+                //                // fill back column ref in hierarchy
+                //                if (dim.isHierarchy()) {
+                //                    for (int i = 0; i < dimCols.size(); i++)
+                //                        dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
+                //                }
             }
 
-            TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
+            TblColRef[] dimColArray = dimCols.toArray(new TblColRef[dimCols.size()]);
             dim.setColumnRefs(dimColArray);
 
             // init derived columns
-            TblColRef[] hostCols = dimColArray;
             if (dim.isDerived()) {
                 String[] derived = dim.getDerived();
                 String[][] split = splitDerivedColumnAndExtra(derived);
@@ -517,7 +507,7 @@ public class CubeDesc extends RootPersistentEntity {
                 for (int i = 0; i < derivedNames.length; i++) {
                     derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
                 }
-                initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+                initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
             }
 
             // PK-FK derive the other side
@@ -528,10 +518,10 @@ public class CubeDesc extends RootPersistentEntity {
                 allColumns.addAll(Arrays.asList(fk));
                 allColumns.addAll(Arrays.asList(pk));
                 for (int i = 0; i < fk.length; i++) {
-                    int find = ArrayUtils.indexOf(hostCols, fk[i]);
+                    int find = ArrayUtils.indexOf(dimColArray, fk[i]);
                     if (find >= 0) {
                         TblColRef derivedCol = initDimensionColRef(pk[i]);
-                        initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
+                        initDerivedMap(dimColArray[find], DeriveType.PK_FK, dim, derivedCol);
                     }
                 }
                 /** disable this code as we don't need fk be derived from pk
@@ -693,7 +683,7 @@ public class CubeDesc extends RootPersistentEntity {
         for (int i = 0; i < measures.size(); i++)
             measureIndexLookup.put(measures.get(i).getName(), i);
 
-        for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
+        for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) {
             for (HBaseColumnDesc c : cf.getColumns()) {
                 String[] colMeasureRefs = c.getMeasureRefs();
                 MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length];
@@ -709,53 +699,6 @@ public class CubeDesc extends RootPersistentEntity {
         }
     }
 
-    private void sortDimAndMeasure() {
-        sortDimensionsByID();
-        sortMeasuresByID();
-        for (DimensionDesc dim : dimensions) {
-            sortHierarchiesByLevel(dim.getHierarchy());
-        }
-    }
-
-    private void sortDimensionsByID() {
-        Collections.sort(dimensions, new Comparator<DimensionDesc>() {
-            @Override
-            public int compare(DimensionDesc d1, DimensionDesc d2) {
-                Integer id1 = d1.getId();
-                Integer id2 = d2.getId();
-                return id1.compareTo(id2);
-            }
-        });
-    }
-
-    private void sortMeasuresByID() {
-        if (measures == null) {
-            measures = Lists.newArrayList();
-        }
-
-        Collections.sort(measures, new Comparator<MeasureDesc>() {
-            @Override
-            public int compare(MeasureDesc m1, MeasureDesc m2) {
-                Integer id1 = m1.getId();
-                Integer id2 = m2.getId();
-                return id1.compareTo(id2);
-            }
-        });
-    }
-
-    private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
-        if (hierarchies != null) {
-            Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
-                @Override
-                public int compare(HierarchyDesc h1, HierarchyDesc h2) {
-                    Integer level1 = Integer.parseInt(h1.getLevel());
-                    Integer level2 = Integer.parseInt(h2.getLevel());
-                    return level1.compareTo(level2);
-                }
-            });
-        }
-    }
-
     public boolean hasHolisticCountDistinctMeasures() {
         for (MeasureDesc measure : measures) {
             if (measure.getFunction().isHolisticCountDistinct()) {
@@ -822,7 +765,7 @@ public class CubeDesc extends RootPersistentEntity {
         return storageType;
     }
 
-    void setStorageType(int storageType) {
+    public void setStorageType(int storageType) {
         this.storageType = storageType;
     }
 
@@ -830,7 +773,7 @@ public class CubeDesc extends RootPersistentEntity {
         return engineType;
     }
 
-    void setEngineType(int engineType) {
+    public void setEngineType(int engineType) {
         this.engineType = engineType;
     }
 
@@ -855,7 +798,6 @@ public class CubeDesc extends RootPersistentEntity {
         return measureDisplayColumns;
     }
 
-
     public boolean hasMeasureUsingDictionary() {
         for (MeasureDesc measureDesc : this.getMeasures()) {
             if (measureDesc.getFunction().isTopN())
@@ -864,4 +806,5 @@ public class CubeDesc extends RootPersistentEntity {
 
         return false;
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index e75a123..bccae58 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -18,11 +18,10 @@
 
 package org.apache.kylin.cube.model;
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.LookupDesc;
@@ -32,34 +31,27 @@ import org.apache.kylin.metadata.model.TblColRef;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
 
 /**
  */
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
 public class DimensionDesc {
 
-    @JsonProperty("id")
-    private int id;
-
     @JsonProperty("name")
     private String name;
-
-    @JsonProperty("hierarchy")
-    private boolean isHierarchy;
     @JsonProperty("table")
     private String table;
     @JsonProperty("column")
-    private String[] column;
+    private String column;
     @JsonProperty("derived")
     private String[] derived;
 
     private TableDesc tableDesc;
     private JoinDesc join;
-    private HierarchyDesc[] hierarchy;
 
     // computed
     private TblColRef[] columnRefs;
-    private TblColRef[] derivedColRefs;
 
     public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
         if (name != null)
@@ -80,29 +72,30 @@ public class DimensionDesc {
             }
         }
 
-        if (isHierarchy && this.column.length > 0) {
-            List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
-            for (int i = 0, n = this.column.length; i < n; i++) {
-                String aColumn = this.column[i];
-                HierarchyDesc aHierarchy = new HierarchyDesc();
-                aHierarchy.setLevel(String.valueOf(i + 1));
-                aHierarchy.setColumn(aColumn);
-                hierarchyList.add(aHierarchy);
-            }
-
-            this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
-        }
+        //        if (isHierarchy && this.column.length > 0) {
+        //            List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
+        //            for (int i = 0, n = this.column.length; i < n; i++) {
+        //                String aColumn = this.column[i];
+        //                HierarchyDesc aHierarchy = new HierarchyDesc();
+        //                aHierarchy.setLevel(String.valueOf(i + 1));
+        //                aHierarchy.setColumn(aColumn);
+        //                hierarchyList.add(aHierarchy);
+        //            }
+        //
+        //            this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
+        //        }
+        //
+        //        if (hierarchy != null && hierarchy.length == 0)
+        //            hierarchy = null;
+
+        //        if (hierarchy != null) {
+        //            for (HierarchyDesc h : hierarchy)
+        //                h.setColumn(h.getColumn().toUpperCase());
+        //        }
 
-        if (hierarchy != null && hierarchy.length == 0)
-            hierarchy = null;
         if (derived != null && derived.length == 0)
             derived = null;
 
-        if (hierarchy != null) {
-            for (HierarchyDesc h : hierarchy)
-                h.setColumn(h.getColumn().toUpperCase());
-        }
-
         if (derived != null) {
             StringUtil.toUpperCaseArray(derived, derived);
         }
@@ -112,29 +105,10 @@ public class DimensionDesc {
         }
     }
 
-    public boolean isHierarchyColumn(TblColRef col) {
-        if (hierarchy == null)
-            return false;
-
-        for (HierarchyDesc hier : hierarchy) {
-            if (hier.getColumnRef().equals(col))
-                return true;
-        }
-        return false;
-    }
-
     public boolean isDerived() {
         return derived != null;
     }
 
-    public boolean isHierarchy() {
-        return isHierarchy;
-    }
-
-    public void setHierarchy(boolean isHierarchy) {
-        this.isHierarchy = isHierarchy;
-    }
-
     public String getTable() {
         return table;
     }
@@ -143,14 +117,6 @@ public class DimensionDesc {
         this.table = table;
     }
 
-    public int getId() {
-        return id;
-    }
-
-    public void setId(int id) {
-        this.id = id;
-    }
-
     public JoinDesc getJoin() {
         return join;
     }
@@ -171,22 +137,14 @@ public class DimensionDesc {
         this.columnRefs = colRefs;
     }
 
-    public String[] getColumn() {
+    public String getColumn() {
         return this.column;
     }
 
-    public void setColumn(String[] column) {
+    public void setColumn(String column) {
         this.column = column;
     }
 
-    public HierarchyDesc[] getHierarchy() {
-        return hierarchy;
-    }
-
-    public void setHierarchy(HierarchyDesc[] hierarchy) {
-        this.hierarchy = hierarchy;
-    }
-
     public String[] getDerived() {
         return derived;
     }
@@ -195,45 +153,22 @@ public class DimensionDesc {
         this.derived = derived;
     }
 
-    public TblColRef[] getDerivedColRefs() {
-        return derivedColRefs;
-    }
-
-    public void setDerivedColRefs(TblColRef[] derivedColRefs) {
-        this.derivedColRefs = derivedColRefs;
-    }
-
     public TableDesc getTableDesc() {
         return this.tableDesc;
     }
 
     @Override
     public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-
-        DimensionDesc that = (DimensionDesc) o;
-
-        if (id != that.id)
-            return false;
-        if (!name.equals(that.name))
-            return false;
-
-        return true;
+        throw new NotImplementedException();
     }
 
     @Override
     public int hashCode() {
-        int result = id;
-        result = 31 * result + name.hashCode();
-        return result;
+        throw new NotImplementedException();
     }
 
     @Override
     public String toString() {
-        return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
+        return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString();
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index e390bd6..2ef1e17 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -43,7 +43,7 @@ public class HBaseMappingDesc {
 
     public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
         Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
-        HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
+        HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping();
         if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
             return result;
         }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 278b59f..7b12cd7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -18,11 +18,15 @@
 
 package org.apache.kylin.cube.model;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
 
 /**
  * @author yangli9
@@ -31,37 +35,74 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
 public class RowKeyColDesc {
 
+    public enum ColEncodingType {
+        DICT, FIXED_LEN
+    }
+
+    public class ColEncoding {
+        public ColEncodingType type;
+        public Object param;
+
+        public ColEncoding(ColEncodingType type, Object param) {
+            this.type = type;
+            this.param = param;
+        }
+    }
+
     @JsonProperty("column")
     private String column;
-    @JsonProperty("length")
-    private int length;
-    @JsonProperty("dictionary")
-    private String dictionary;
-    @JsonProperty("mandatory")
-    private boolean mandatory = false;
+    @JsonProperty("encoding")
+    private String encoding;
 
     // computed
+    private ColEncoding colEncoding;
     private int bitIndex;
     private TblColRef colRef;
 
-    public String getDictionary() {
-        return dictionary;
+    public void init() {
+
+        //dict or fix length?
+        Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
+        if (this.encoding.equalsIgnoreCase("dict")) {
+            this.colEncoding = new ColEncoding(ColEncodingType.DICT, null);
+        } else if (this.encoding.startsWith("fixed_length")) {
+            int length = RowConstants.ROWKEY_COL_DEFAULT_LENGTH;
+            if (this.encoding.indexOf(":") > 0) {
+                length = Integer.parseInt(this.encoding.substring(this.encoding.indexOf(":") + 1));
+            }
+            this.colEncoding = new ColEncoding(ColEncodingType.FIXED_LEN, length);
+        } else {
+            throw new IllegalArgumentException("Not supported row key col encoding:" + this.encoding);
+        }
+    }
+
+    public String getEncoding() {
+        return encoding;
+    }
+
+    public void setEncoding(String encoding) {
+        this.encoding = encoding;
     }
 
     public String getColumn() {
         return column;
     }
 
-    void setColumn(String column) {
+    public void setColumn(String column) {
         this.column = column;
     }
 
-    public int getLength() {
-        return length;
+    public boolean isUsingDictionary() {
+        return this.colEncoding.type == ColEncodingType.DICT;
+
     }
 
-    public boolean isMandatory() {
-        return mandatory;
+    public int getLength() {
+        if (this.colEncoding.type == ColEncodingType.FIXED_LEN) {
+            return (Integer) this.colEncoding.param;
+        } else {
+            throw new IllegalStateException("Not knowing the col's length");
+        }
     }
 
     public int getBitIndex() {
@@ -80,13 +121,9 @@ public class RowKeyColDesc {
         this.colRef = colRef;
     }
 
-    public void setDictionary(String dictionary) {
-        this.dictionary = dictionary;
-    }
-
     @Override
     public String toString() {
-        return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
+        return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
     }
 
 }


Mime
View raw message