kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [34/48] kylin git commit: KYLIN-2783 Refactor CuboidScheduler to be extensible
Date Fri, 15 Sep 2017 14:42:54 GMT
KYLIN-2783 Refactor CuboidScheduler to be extensible

Signed-off-by: Li Yang <liyang@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/465c5070
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/465c5070
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/465c5070

Branch: refs/heads/ranger
Commit: 465c5070dcc27c6d42c2590cf0685ab154b40e7b
Parents: 084a6fb
Author: Zhong <nju_yaho@apache.org>
Authored: Thu Aug 17 20:17:26 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Sun Sep 10 07:47:20 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeDescManager.java  |   3 -
 .../org/apache/kylin/cube/CubeInstance.java     |  18 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  17 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   5 +
 .../kylin/cube/common/RowKeySplitter.java       |   4 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java    | 180 +++--------
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  20 +-
 .../kylin/cube/cuboid/CuboidScheduler.java      |  25 +-
 .../cube/cuboid/DefaultCuboidScheduler.java     | 162 +++++++++-
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  | 322 +++++++++++++++++++
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  17 +-
 .../cube/inmemcubing/DoggedCubeBuilder.java     |   9 +-
 .../cube/inmemcubing/InMemCubeBuilder.java      |  11 +-
 .../org/apache/kylin/cube/kv/RowKeyDecoder.java |   4 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  11 +-
 .../kylin/cube/model/AggregationGroup.java      |   4 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  33 +-
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   3 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  26 +-
 .../apache/kylin/cube/cuboid/CuboidTest.java    |  60 ++--
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java |   2 +-
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java |   6 +-
 .../gtrecord/GTCubeStorageQueryBase.java        |   6 +-
 .../kylin/storage/translate/HBaseKeyRange.java  |   2 +-
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   2 +-
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   2 +-
 .../engine/mr/common/BaseCuboidBuilder.java     |  14 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   7 +-
 .../mr/steps/FactDistinctColumnsMapper.java     |   2 +-
 .../engine/mr/steps/InMemCuboidMapper.java      |   3 +-
 .../kylin/engine/mr/steps/KVGTRecordWriter.java |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |   2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |   6 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   9 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  12 +-
 .../spark/cube/DefaultTupleConverter.java       |   2 +-
 .../ITDoggedCubeBuilderStressTest.java          |   2 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java    |   4 +-
 .../inmemcubing/ITInMemCubeBuilderTest.java     |   2 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   4 +-
 .../cube/MeasureTypeOnlyAggrInBaseTest.java     |   4 +-
 42 files changed, 738 insertions(+), 293 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 6635366..f49c9be 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -171,7 +171,6 @@ public class CubeDescManager {
         CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false);
 
         cubeDescMap.putLocal(ndesc.getName(), ndesc);
-        Cuboid.reloadCache(name);
 
         // if related cube is in DESCBROKEN state before, change it back to DISABLED
         CubeManager cubeManager = CubeManager.getInstance(config);
@@ -297,13 +296,11 @@ public class CubeDescManager {
         String path = cubeDesc.getResourcePath();
         getStore().deleteResource(path);
         cubeDescMap.remove(cubeDesc.getName());
-        Cuboid.reloadCache(cubeDesc.getName());
     }
 
     // remove cubeDesc
     public void removeLocalCubeDesc(String name) throws IOException {
         cubeDescMap.removeLocal(name);
-        Cuboid.reloadCache(name);
     }
 
     private void reloadAllCubeDesc() throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fad942c..246cbf6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -95,11 +96,26 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     @JsonProperty("create_time_utc")
     private long createTimeUTC;
 
+    // cuboid scheduler lazy built
+    transient private CuboidScheduler cuboidScheduler;
+
     // default constructor for jackson
     public CubeInstance() {
     }
 
-    public Segments<CubeSegment> getBuildingSegments() {
+    public CuboidScheduler getCuboidScheduler() {
+        if (cuboidScheduler != null)
+            return cuboidScheduler;
+
+        synchronized (this) {
+            if (cuboidScheduler == null) {
+                cuboidScheduler = getDescriptor().getInitialCuboidScheduler();
+            }
+        }
+        return cuboidScheduler;
+    }
+
+    public List<CubeSegment> getBuildingSegments() {
         return segments.getBuildingSegments();
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b782a5e..6ebfd99 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -43,6 +43,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.dict.DictionaryInfo;
@@ -306,13 +307,14 @@ public class CubeManager implements IRealizationProvider {
         // delete cube instance and cube desc
         CubeInstance cube = getCube(cubeName);
 
-        if (deleteDesc && cube.getDescriptor() != null) {
-            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
-        }
-
         // remove cube and update cache
         getStore().deleteResource(cube.getResourcePath());
         cubeMap.remove(cube.getName());
+        Cuboid.reloadCache(cube);
+
+        if (deleteDesc && cube.getDescriptor() != null) {
+            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+        }
 
         // delete cube from project
         ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
@@ -629,17 +631,20 @@ public class CubeManager implements IRealizationProvider {
      * @param cubeName
      */
     public CubeInstance reloadCubeLocal(String cubeName) {
-        return reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+        CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+        Cuboid.reloadCache(cubeInstance);
+        return cubeInstance;
     }
 
     public void removeCubeLocal(String cubeName) {
         CubeInstance cube = cubeMap.get(cubeName);
         if (cube != null) {
+            cubeMap.removeLocal(cubeName);
             for (CubeSegment segment : cube.getSegments()) {
                 usedStorageLocation.remove(segment.getUuid());
             }
+            Cuboid.reloadCache(cube);
         }
-        cubeMap.removeLocal(cubeName);
     }
 
     public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 358183e..495f9c0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ShardingHash;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -127,6 +128,10 @@ public class CubeSegment implements IBuildable, ISegment, Serializable {
         return getCubeInstance().getDescriptor();
     }
 
+    public CuboidScheduler getCuboidScheduler() {
+        return getCubeInstance().getCuboidScheduler();
+    }
+
     public static String makeSegmentName(TSRange tsRange, SegmentRange segRange) {
         if (tsRange == null && segRange == null) {
             return "FULL_BUILD";

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index acebce4..cd26347 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 
 public class RowKeySplitter implements java.io.Serializable {
 
+    private CubeSegment cubeSegment;
     private CubeDesc cubeDesc;
     private RowKeyColumnIO colIO;
 
@@ -63,6 +64,7 @@ public class RowKeySplitter implements java.io.Serializable {
     }
 
     public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
+        this.cubeSegment = cubeSeg;
         this.enableSharding = cubeSeg.isEnableSharding();
         this.cubeDesc = cubeSeg.getCubeDesc();
         IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);
@@ -113,7 +115,7 @@ public class RowKeySplitter implements java.io.Serializable {
         offset += RowConstants.ROWKEY_CUBOIDID_LEN;
 
         long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
-        Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId);
+        Cuboid cuboid = Cuboid.findById(cubeSegment, lastSplittedCuboidId);
 
         // rowkey columns
         for (int i = 0; i < cuboid.getColumns().size(); i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index b71608c..fa83775 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -21,15 +21,15 @@ package org.apache.kylin.cube.cuboid;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
@@ -38,16 +38,15 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 @SuppressWarnings("serial")
 public class Cuboid implements Comparable<Cuboid>, Serializable {
 
-    private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
+    //TODO Guava cache may be better
+    private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap();
 
     // smaller is better
     public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
@@ -57,10 +56,22 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         }
     };
 
-    // this is the only entry point for query to find the right cuboid
-    public static Cuboid identifyCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
-        long cuboidID = identifyCuboidId(cubeDesc, dimensions, metrics);
-        return Cuboid.findById(cubeDesc, cuboidID);
+    // this is the only entry point for query to find the right cuboid for a segment
+    public static Cuboid identifyCuboid(CubeSegment cubeSegment, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        return identifyCuboid(cubeSegment.getCuboidScheduler(), dimensions, metrics);
+    }
+
+    // this is the only entry point for query to find the right cuboid for a cube instance
+    public static Cuboid identifyCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        return identifyCuboid(cubeInstance.getCuboidScheduler(), dimensions, metrics);
+    }
+
+    public static Cuboid identifyCuboid(CuboidScheduler cuboidScheduler, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        long cuboidID = identifyCuboidId(cuboidScheduler.getCubeDesc(), dimensions, metrics);
+        return Cuboid.findById(cuboidScheduler, cuboidID);
     }
 
     public static long identifyCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -77,140 +88,49 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         return cuboidID;
     }
 
-    // for full cube, no need to translate cuboid
-    public static Cuboid findForFullCube(CubeDesc cube, long cuboidID) {
+    // for mandatory cuboid, no need to translate cuboid
+    public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) {
         return new Cuboid(cube, cuboidID, cuboidID);
     }
 
-    public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
-        return findById(cube, Bytes.toLong(cuboidID));
+    public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) {
+        return findById(cuboidScheduler, Bytes.toLong(cuboidID));
     }
 
-    public static Cuboid findById(CubeDesc cube, long cuboidID) {
-        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cube.getName());
+    public static Cuboid findById(CubeSegment cubeSegment, long cuboidID) {
+        return findById(cubeSegment.getCuboidScheduler(), cuboidID);
+    }
+
+    public static Cuboid findById(CubeInstance cubeInstance, long cuboidID) {
+        return findById(cubeInstance.getCuboidScheduler(), cuboidID);
+    }
+
+    @VisibleForTesting
+    static Cuboid findById(CubeDesc cubeDesc, long cuboidID) {
+        return findById(cubeDesc.getInitialCuboidScheduler(), cuboidID);
+    }
+
+    public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) {
+        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey());
         if (cubeCache == null) {
-            cubeCache = new ConcurrentHashMap<Long, Cuboid>();
-            CUBOID_CACHE.put(cube.getName(), cubeCache);
+            cubeCache = Maps.newConcurrentMap();
+            CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache);
         }
         Cuboid cuboid = cubeCache.get(cuboidID);
         if (cuboid == null) {
-            long validCuboidID = translateToValidCuboid(cube, cuboidID);
-            cuboid = new Cuboid(cube, cuboidID, validCuboidID);
+            long validCuboidID = cuboidScheduler.findBestMatchCuboid(cuboidID);
+            cuboid = new Cuboid(cuboidScheduler.getCubeDesc(), cuboidID, validCuboidID);
             cubeCache.put(cuboidID, cuboid);
         }
         return cuboid;
     }
 
-    public static boolean isValid(CubeDesc cube, long cuboidID) {
-        return cube.getAllCuboids().contains(cuboidID);
-    }
-
     public static long getBaseCuboidId(CubeDesc cube) {
         return cube.getRowkey().getFullMask();
     }
 
     public static Cuboid getBaseCuboid(CubeDesc cube) {
-        return findById(cube, getBaseCuboidId(cube));
-    }
-
-    static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
-        long baseCuboidId = getBaseCuboidId(cubeDesc);
-        if (cubeDesc.getAllCuboids().contains(cuboidID)) {
-            return cuboidID;
-        }
-        List<Long> onTreeCandidates = Lists.newArrayList();
-        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
-            Long candidate = translateToOnTreeCuboid(agg, cuboidID);
-            if (candidate != null) {
-                onTreeCandidates.add(candidate);
-            }
-        }
-
-        if (onTreeCandidates.size() == 0) {
-            return baseCuboidId;
-        }
-
-        long onTreeCandi = Collections.min(onTreeCandidates, cuboidSelectComparator);
-        if (isValid(cubeDesc, onTreeCandi)) {
-            return onTreeCandi;
-        }
-
-        return cubeDesc.getCuboidScheduler().findBestMatchCuboid(onTreeCandi);
-    }
-
-    private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
-        if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
-            //the partial cube might not contain all required dims
-            return null;
-        }
-
-        // add mandantory
-        cuboidID = cuboidID | agg.getMandatoryColumnMask();
-
-        // add hierarchy
-        for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
-            long fullMask = hierarchyMask.fullMask;
-            long intersect = cuboidID & fullMask;
-            if (intersect != 0 && intersect != fullMask) {
-
-                boolean startToFill = false;
-                for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
-                    if (startToFill) {
-                        cuboidID |= hierarchyMask.dims[i];
-                    } else {
-                        if ((cuboidID & hierarchyMask.dims[i]) != 0) {
-                            startToFill = true;
-                            cuboidID |= hierarchyMask.dims[i];
-                        }
-                    }
-                }
-            }
-        }
-
-        // add joint dims
-        for (Long joint : agg.getJoints()) {
-            if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
-                cuboidID = cuboidID | joint;
-            }
-        }
-
-        if (!agg.isOnTree(cuboidID)) {
-            // no column, add one column
-            long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), agg.getJoints());
-            if (nonJointDims != 0) {
-                long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
-                    @Override
-                    public Long apply(HierarchyMask input) {
-                        return input.fullMask;
-                    }
-                }));
-                if (nonJointNonHierarchy != 0) {
-                    //there exists dim that does not belong to any joint or any hierarchy, that's perfect
-                    return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
-                } else {
-                    //choose from a hierarchy that does not intersect with any joint dim, only check level 1 
-                    long allJointDims = agg.getJointDimsMask();
-                    for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
-                        long dim = hierarchyMask.allMasks[0];
-                        if ((dim & allJointDims) == 0) {
-                            return cuboidID | dim;
-                        }
-                    }
-                }
-            }
-
-            cuboidID = cuboidID | Collections.min(agg.getJoints(), cuboidSelectComparator);
-            Preconditions.checkState(agg.isOnTree(cuboidID));
-        }
-        return cuboidID;
-    }
-
-    private static long removeBits(long original, Collection<Long> toRemove) {
-        long ret = original;
-        for (Long joint : toRemove) {
-            ret = ret & ~joint;
-        }
-        return ret;
+        return findForMandatory(cube, getBaseCuboidId(cube));
     }
 
     // ============================================================================
@@ -310,8 +230,12 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         CUBOID_CACHE.clear();
     }
 
-    public static void reloadCache(String cubeDescName) {
-        CUBOID_CACHE.remove(cubeDescName);
+    public static void reloadCache(CubeInstance cubeInstance) {
+        reloadCache(cubeInstance.getCuboidScheduler());
+    }
+
+    private static void reloadCache(CuboidScheduler cuboidScheduler) {
+        CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index d657a43..4300a6c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -44,7 +44,7 @@ public class CuboidCLI {
     }
 
     public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
-        CuboidScheduler scheduler = cubeDesc.getCuboidScheduler();
+        CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler();
         long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
         Collection<Long> cuboidSet = new TreeSet<Long>();
         cuboidSet.add(baseCuboid);
@@ -88,28 +88,28 @@ public class CuboidCLI {
                 //check all valid and invalid
                 for (long i = 0; i < baseCuboid; ++i) {
                     if (cuboidSet.contains(i)) {
-                        if (!Cuboid.isValid(cubeDesc, i)) {
+                        if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
+                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) != i) {
                             throw new RuntimeException();
                         }
                     } else {
-                        if (Cuboid.isValid(cubeDesc, i)) {
+                        if (cubeDesc.getInitialCuboidScheduler().isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
+                        long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i);
                         if (corrected == i) {
                             throw new RuntimeException();
                         }
 
-                        if (!Cuboid.isValid(cubeDesc, corrected)) {
+                        if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
+                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected) != corrected) {
                             throw new RuntimeException();
                         }
                     }
@@ -125,7 +125,7 @@ public class CuboidCLI {
         long baseCuboid = Cuboid.getBaseCuboidId(cube);
         TreeSet<Long> expectedCuboids = new TreeSet<Long>();
         for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
-            if (Cuboid.isValid(cube, cuboid)) {
+            if (cube.getInitialCuboidScheduler().isValid(cuboid)) {
                 expectedCuboids.add(cuboid);
             }
         }
@@ -133,10 +133,10 @@ public class CuboidCLI {
     }
 
     public static int[] calculateAllLevelCount(CubeDesc cube) {
-        int levels = cube.getBuildLevel();
+        int levels = cube.getInitialBuildLevel();
         int[] allLevelCounts = new int[levels + 1];
 
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
         LinkedList<Long> nextQueue = new LinkedList<Long>();
         LinkedList<Long> currentQueue = new LinkedList<Long>();
         long baseCuboid = Cuboid.getBaseCuboidId(cube);

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index e802230..cef0f77 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
 
 import com.google.common.base.Preconditions;
@@ -63,14 +62,25 @@ abstract public class CuboidScheduler {
     
     /** Returns a cuboid on the tree that best matches the request cuboid. */
     abstract public long findBestMatchCuboid(long requestCuboid);
-    
-    /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */
-    abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg);
+
+    /** Returns whether requestCuboid is valid or not*/
+    abstract public boolean isValid(long requestCuboid);
+
+    /** Returns the key for what this cuboid scheduler responsible for*/
+    abstract public String getResponsibleKey();
 
     // ============================================================================
     
     private transient List<List<Long>> cuboidsByLayer;
 
+    public long getBaseCuboidId() {
+        return Cuboid.getBaseCuboidId(cubeDesc);
+    }
+
+    public CubeDesc getCubeDesc() {
+        return cubeDesc;
+    }
+
     /**
      * Get cuboids by layer. It's built from pre-expanding tree.
      * @return layered cuboids
@@ -105,4 +115,11 @@ abstract public class CuboidScheduler {
         return cuboidsByLayer;
     }
 
+    /**
+     * Get cuboid level count except base cuboid
+     * @return
+     */
+    public int getBuildLevel() {
+        return getCuboidsByLayer().size() - 1;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index f7d22da..cba417a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -35,10 +35,14 @@ import javax.annotation.Nullable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.TooManyCuboidException;
 
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -83,14 +87,48 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return Sets.newHashSet(allCuboidIds);
     }
 
+    @Override
+    public boolean isValid(long requestCuboid) {
+        return allCuboidIds.contains(requestCuboid);
+    }
+
     /**
-     * Get the parent cuboid really on the spanning tree.
-     * @param child an on-tree cuboid
+     * Get the parent cuboid rely on the spanning tree.
+     * @param cuboid an on-tree cuboid
      * @return
      */
     @Override
-    public long findBestMatchCuboid(long child) {
-        long parent = getOnTreeParent(child);
+    public long findBestMatchCuboid(long cuboid) {
+        return findBestMatchCuboid1(cuboid);
+    }
+
+    public long findBestMatchCuboid1(long cuboid) {
+        if (isValid(cuboid)) {
+            return cuboid;
+        }
+
+        List<Long> onTreeCandidates = Lists.newArrayList();
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            Long candidate = translateToOnTreeCuboid(agg, cuboid);
+            if (candidate != null) {
+                onTreeCandidates.add(candidate);
+            }
+        }
+
+        if (onTreeCandidates.size() == 0) {
+            return getBaseCuboidId();
+        }
+
+        long onTreeCandi = Collections.min(onTreeCandidates, Cuboid.cuboidSelectComparator);
+        if (isValid(onTreeCandi)) {
+            return onTreeCandi;
+        }
+
+        return doFindBestMatchCuboid1(onTreeCandi);
+    }
+
+    public long doFindBestMatchCuboid1(long cuboid) {
+        long parent = getOnTreeParent(cuboid);
         while (parent > 0) {
             if (cubeDesc.getAllCuboids().contains(parent)) {
                 break;
@@ -99,11 +137,88 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         }
 
         if (parent <= 0) {
-            throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+            throw new IllegalStateException("Can't find valid parent for Cuboid " + cuboid);
         }
         return parent;
     }
 
+    private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
+        if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+            //the partial cube might not contain all required dims
+            return null;
+        }
+
+        // add mandantory
+        cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+        // add hierarchy
+        for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+            long fullMask = hierarchyMask.fullMask;
+            long intersect = cuboidID & fullMask;
+            if (intersect != 0 && intersect != fullMask) {
+
+                boolean startToFill = false;
+                for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
+                    if (startToFill) {
+                        cuboidID |= hierarchyMask.dims[i];
+                    } else {
+                        if ((cuboidID & hierarchyMask.dims[i]) != 0) {
+                            startToFill = true;
+                            cuboidID |= hierarchyMask.dims[i];
+                        }
+                    }
+                }
+            }
+        }
+
+        // add joint dims
+        for (Long joint : agg.getJoints()) {
+            if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+                cuboidID = cuboidID | joint;
+            }
+        }
+
+        if (!agg.isOnTree(cuboidID)) {
+            // no column, add one column
+            long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()),
+                    agg.getJoints());
+            if (nonJointDims != 0) {
+                long nonJointNonHierarchy = removeBits(nonJointDims,
+                        Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+                            @Override
+                            public Long apply(HierarchyMask input) {
+                                return input.fullMask;
+                            }
+                        }));
+                if (nonJointNonHierarchy != 0) {
+                    //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+                    return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
+                } else {
+                    //choose from a hierarchy that does not intersect with any joint dim, only check level 1
+                    long allJointDims = agg.getJointDimsMask();
+                    for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+                        long dim = hierarchyMask.allMasks[0];
+                        if ((dim & allJointDims) == 0) {
+                            return cuboidID | dim;
+                        }
+                    }
+                }
+            }
+
+            cuboidID = cuboidID | Collections.min(agg.getJoints(), Cuboid.cuboidSelectComparator);
+            Preconditions.checkState(agg.isOnTree(cuboidID));
+        }
+        return cuboidID;
+    }
+
+    private static long removeBits(long original, Collection<Long> toRemove) {
+        long ret = original;
+        for (Long joint : toRemove) {
+            ret = ret & ~joint;
+        }
+        return ret;
+    }
+
     private long getOnTreeParent(long child) {
         Collection<Long> candidates = getOnTreeParents(child);
         if (candidates == null || candidates.isEmpty()) {
@@ -259,7 +374,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
      * @param agg agg group
      * @return cuboidId list
      */
-    @Override
     public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) {
         Set<Long> cuboidHolder = new HashSet<>();
 
@@ -352,4 +466,40 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return Long.bitCount(cuboidID) <= dimCap;
     }
 
+    public long findBestMatchCuboid2(long cuboid) {
+        long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc));
+        if (bestParent < -1) {
+            throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid);
+        }
+        return bestParent;
+    }
+
+    private long doFindBestMatchCuboid2(long cuboid, long parent) {
+        if (!canDerive(cuboid, parent)) {
+            return -1;
+        }
+        List<Long> children = parent2child.get(parent);
+        List<Long> candidates = Lists.newArrayList();
+        if (children != null) {
+            for (long child : children) {
+                long candidate = doFindBestMatchCuboid2(cuboid, child);
+                if (candidate > 0) {
+                    candidates.add(candidate);
+                }
+            }
+        }
+        if (candidates.isEmpty()) {
+            candidates.add(parent);
+        }
+
+        return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+    }
+
+    private boolean canDerive(long cuboidId, long parentCuboid) {
+        return (cuboidId & ~parentCuboid) == 0;
+    }
+
+    public String getResponsibleKey() {
+        return CubeDesc.class.getName() + "-" + cubeDesc.getName();
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
new file mode 100644
index 0000000..414fc4a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.model.CubeDesc;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class TreeCuboidScheduler extends CuboidScheduler {
+
+    final private CuboidTree cuboidTree;
+
+    public TreeCuboidScheduler(CubeDesc cubeDesc, List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+        super(cubeDesc);
+        cuboidTree = CuboidTree.createFromCuboids(allCuboidIds, cuboidComparator);
+    }
+
+    @Override
+    public Set<Long> getAllCuboidIds() {
+        return cuboidTree.getAllCuboidIds();
+    }
+
+    @Override
+    public int getCuboidCount() {
+        return cuboidTree.getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
+    }
+
+    @Override
+    public List<Long> getSpanningCuboid(long cuboidId) {
+        return cuboidTree.getSpanningCuboid(cuboidId);
+    }
+
+    @Override
+    public long findBestMatchCuboid(long cuboidId) {
+        return cuboidTree.findBestMatchCuboid(cuboidId);
+    }
+
+    @Override
+    public boolean isValid(long requestCuboid) {
+        return cuboidTree.isValid(requestCuboid);
+    }
+
+    public static class CuboidTree {
+        private int treeLevels;
+
+        private TreeNode root;
+
+        private Comparator<Long> cuboidComparator;
+
+        private Map<Long, TreeNode> index = new HashMap<>();
+
+        @VisibleForTesting
+        static CuboidTree createFromCuboids(List<Long> allCuboidIds) {
+            return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator);
+        }
+
+        @VisibleForTesting
+        static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+            // sort the cuboid ids in descending order, so that don't need to adjust
+            // the cuboid tree when adding cuboid id to the tree.
+            Collections.sort(allCuboidIds, new Comparator<Long>() {
+                @Override
+                public int compare(Long o1, Long o2) {
+                    return Long.compare(o2, o1);
+                }
+            });
+            long basicCuboidId = allCuboidIds.get(0);
+            CuboidTree cuboidTree = new CuboidTree(cuboidComparator);
+            cuboidTree.setRoot(basicCuboidId);
+
+            for (long cuboidId : allCuboidIds) {
+                cuboidTree.addCuboid(cuboidId);
+            }
+            cuboidTree.buildIndex();
+            return cuboidTree;
+        }
+
+        private CuboidTree(Comparator<Long> cuboidComparator) {
+            this.cuboidComparator = cuboidComparator;
+        }
+
+        public Set<Long> getAllCuboidIds() {
+            return index.keySet();
+        }
+
+        public List<Long> getSpanningCuboid(long cuboidId) {
+            TreeNode node = index.get(cuboidId);
+            if (node == null) {
+                throw new IllegalArgumentException("the cuboid:" + cuboidId + " is not exist in the tree");
+            }
+
+            return Lists.transform(node.children, new Function<TreeNode, Long>() {
+                @Nullable
+                @Override
+                public Long apply(@Nullable TreeNode input) {
+                    return input.cuboidId;
+                }
+            });
+        }
+
+        public long findBestMatchCuboid(long cuboidId) {
+            // exactly match
+            if (isValid(cuboidId)) {
+                return cuboidId;
+            }
+
+            return findBestParent(cuboidId).cuboidId;
+        }
+
+        public boolean isValid(long cuboidId) {
+            return index.containsKey(cuboidId);
+        }
+
+        private int getCuboidCount(long cuboidId) {
+            int r = 1;
+            for (Long child : getSpanningCuboid(cuboidId)) {
+                r += getCuboidCount(child);
+            }
+            return r;
+        }
+
+        public void print(PrintWriter out) {
+            int dimensionCnt = Long.bitCount(root.cuboidId);
+            doPrint(root, dimensionCnt, 0, out);
+        }
+
+        private void doPrint(TreeNode node, int dimensionCount, int depth, PrintWriter out) {
+            printCuboid(node.cuboidId, dimensionCount, depth, out);
+
+            for (TreeNode child : node.children) {
+                doPrint(child, dimensionCount, depth + 1, out);
+            }
+        }
+
+        private void printCuboid(long cuboidID, int dimensionCount, int depth, PrintWriter out) {
+            StringBuffer sb = new StringBuffer();
+            for (int i = 0; i < depth; i++) {
+                sb.append("    ");
+            }
+            String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount);
+            sb.append("|---- Cuboid ").append(cuboidName).append("(" + cuboidID + ")");
+            out.println(sb.toString());
+        }
+
+        private void setRoot(long basicCuboidId) {
+            this.root = new TreeNode(basicCuboidId, 0);
+            this.treeLevels = 0;
+        }
+
+        private void buildIndex() {
+            LinkedList<TreeNode> queue = new LinkedList<>();
+            queue.add(root);
+            while (!queue.isEmpty()) {
+                TreeNode node = queue.removeFirst();
+                index.put(node.cuboidId, node);
+                for (TreeNode child : node.children) {
+                    queue.add(child);
+                }
+            }
+        }
+
+        private void addCuboid(long cuboidId) {
+            TreeNode parent = findBestParent(cuboidId);
+            if (parent != null && parent.cuboidId != cuboidId) {
+                parent.addChild(cuboidId, parent.level);
+                this.treeLevels = Math.max(this.treeLevels, parent.level + 1);
+            }
+        }
+
+        private TreeNode findBestParent(long cuboidId) {
+            TreeNode bestParent = doFindBestParent(cuboidId, root);
+            if (bestParent == null) {
+                throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboidId);
+            }
+            return bestParent;
+        }
+
+        private TreeNode doFindBestParent(long cuboidId, TreeNode parentCuboid) {
+            if (!canDerive(cuboidId, parentCuboid.cuboidId)) {
+                return null;
+            }
+
+            List<TreeNode> candidates = Lists.newArrayList();
+            for (TreeNode childCuboid : parentCuboid.children) {
+                TreeNode candidate = doFindBestParent(cuboidId, childCuboid);
+                if (candidate != null) {
+                    candidates.add(candidate);
+                }
+            }
+            if (candidates.isEmpty()) {
+                candidates.add(parentCuboid);
+            }
+
+            return Collections.min(candidates, new Comparator<TreeNode>() {
+                @Override
+                public int compare(TreeNode o1, TreeNode o2) {
+                    return cuboidComparator.compare(o1.cuboidId, o2.cuboidId);
+                }
+            });
+        }
+
+        private boolean canDerive(long cuboidId, long parentCuboid) {
+            return (cuboidId & ~parentCuboid) == 0;
+        }
+    }
+
+    public static class TreeNode {
+        @JsonProperty("cuboid_id")
+        long cuboidId;
+        @JsonIgnore
+        int level;
+        @JsonProperty("children")
+        List<TreeNode> children = new ArrayList<>();
+
+        public long getCuboidId() {
+            return cuboidId;
+        }
+
+        public int getLevel() {
+            return level;
+        }
+
+        public List<TreeNode> getChildren() {
+            return children;
+        }
+
+        TreeNode(long cuboidId, int level) {
+            this.cuboidId = cuboidId;
+            this.level = level;
+        }
+
+        void addChild(long childId, int parentlevel) {
+            this.children.add(new TreeNode(childId, parentlevel + 1));
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32));
+            result = prime * result + level;
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            TreeNode other = (TreeNode) obj;
+            if (cuboidId != other.cuboidId)
+                return false;
+            if (level != other.level)
+                return false;
+            return true;
+        }
+    }
+
+    /**
+     * Compare cuboid according to the cuboid data row count
+     */
+    public static class CuboidCostComparator implements Comparator<Long> {
+        private Map<Long, Long> cuboidStatistics;
+
+        public CuboidCostComparator(Map<Long, Long> cuboidStatistics) {
+            Preconditions.checkArgument(cuboidStatistics != null,
+                    "the input " + cuboidStatistics + " should not be null!!!");
+            this.cuboidStatistics = cuboidStatistics;
+        }
+
+        @Override
+        public int compare(Long cuboid1, Long cuboid2) {
+            Long rowCnt1 = cuboidStatistics.get(cuboid1);
+            Long rowCnt2 = cuboidStatistics.get(cuboid2);
+            if (rowCnt2 == null || rowCnt1 == null) {
+                return Cuboid.cuboidSelectComparator.compare(cuboid1, cuboid2);
+            }
+            return Long.compare(rowCnt1, rowCnt2);
+        }
+    }
+
+    public String getResponsibleKey() {
+        return CubeInstance.class.getName() + "-" + cubeDesc.getName();
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index c7a4a05..ae38261 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
 import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
@@ -42,6 +43,7 @@ abstract public class AbstractInMemCubeBuilder {
 
     private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class);
 
+    final protected CuboidScheduler cuboidScheduler;
     final protected IJoinedFlatTableDesc flatDesc;
     final protected CubeDesc cubeDesc;
     final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
@@ -49,16 +51,23 @@ abstract public class AbstractInMemCubeBuilder {
     protected int taskThreadCount = 1;
     protected int reserveMemoryMB = 100;
 
-    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        if (flatDesc == null)
+    //    @Deprecated
+    //    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+    //        this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap);
+    //    }
+
+    protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        if (cuboidScheduler == null)
             throw new NullPointerException();
-        if (cubeDesc == null)
+        if (flatDesc == null)
             throw new NullPointerException();
         if (dictionaryMap == null)
             throw new IllegalArgumentException("dictionary cannot be null");
 
+        this.cuboidScheduler = cuboidScheduler;
         this.flatDesc = flatDesc;
-        this.cubeDesc = cubeDesc;
+        this.cubeDesc = cuboidScheduler.getCubeDesc();
         this.dictionaryMap = dictionaryMap;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index a9211da..dd92a2b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -35,7 +35,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequestBuilder;
 import org.apache.kylin.gridtable.IGTScanner;
@@ -57,8 +57,9 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
     private int splitRowThreshold = Integer.MAX_VALUE;
     private int unitRows = 1000;
 
-    public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        super(cubeDesc, flatDesc, dictionaryMap);
+    public DoggedCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        super(cuboidScheduler, flatDesc, dictionaryMap);
 
         // check memory more often if a single row is big
         if (cubeDesc.hasMemoryHungryMeasures())
@@ -276,7 +277,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
         RuntimeException exception;
 
         public SplitThread() {
-            this.builder = new InMemCubeBuilder(cubeDesc, flatDesc, dictionaryMap);
+            this.builder = new InMemCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
             this.builder.setConcurrentThreads(taskThreadCount);
             this.builder.setReserveMemoryMB(reserveMemoryMB);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 93736c9..684c26b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -39,7 +39,6 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.gridtable.CubeGridTable;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
-import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.gridtable.GTAggregateScanner;
 import org.apache.kylin.gridtable.GTBuilder;
 import org.apache.kylin.gridtable.GTInfo;
@@ -71,7 +70,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     private static final double DERIVE_AGGR_CACHE_CONSTANT_FACTOR = 0.1;
     private static final double DERIVE_AGGR_CACHE_VARIABLE_FACTOR = 0.9;
 
-    private final CuboidScheduler cuboidScheduler;
     private final long baseCuboidId;
     private final int totalCuboidCount;
     private final String[] metricsAggrFuncs;
@@ -90,9 +88,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     private Object[] totalSumForSanityCheck;
     private ICuboidCollector resultCollector;
 
-    public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        super(cubeDesc, flatDesc, dictionaryMap);
-        this.cuboidScheduler = cubeDesc.getCuboidScheduler();
+    public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        super(cuboidScheduler, flatDesc, dictionaryMap);
         this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         this.totalCuboidCount = cuboidScheduler.getCuboidCount();
 
@@ -109,8 +107,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
-        GTInfo info = CubeGridTable.newGTInfo(
-                Cuboid.findById(cubeDesc, cuboidID),
+        GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cuboidScheduler, cuboidID),
                 new CubeDimEncMap(cubeDesc, dictionaryMap)
         );
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index d5948d4..5a1f668 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -36,6 +36,7 @@ import org.apache.kylin.metadata.model.TblColRef;
  */
 public class RowKeyDecoder {
 
+    private final CubeSegment cubeSegment;
     private final CubeDesc cubeDesc;
     private final RowKeyColumnIO colIO;
     private final RowKeySplitter rowKeySplitter;
@@ -44,6 +45,7 @@ public class RowKeyDecoder {
     private List<String> values;
 
     public RowKeyDecoder(CubeSegment cubeSegment) {
+        this.cubeSegment = cubeSegment;
         this.cubeDesc = cubeSegment.getCubeDesc();
         this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
         this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
@@ -73,7 +75,7 @@ public class RowKeyDecoder {
         if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
             return;
         }
-        this.cuboid = Cuboid.findById(cubeDesc, cuboidID);
+        this.cuboid = Cuboid.findById(cubeSegment, cuboidID);
     }
 
     private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index a669fb1..40cda76 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -18,7 +18,11 @@
 
 package org.apache.kylin.cube.kv;
 
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
@@ -28,10 +32,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.base.Preconditions;
 
 public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index af026af..852451d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -295,7 +296,8 @@ public class AggregationGroup implements Serializable {
         long combination = 1;
 
         if (this.getDimCap() > 0) {
-            combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size();
+            DefaultCuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+            combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
         } else {
             Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
             Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3cbba50..dff3485 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -51,7 +51,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
@@ -189,8 +189,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
     @JsonInclude(JsonInclude.Include.NON_NULL)
     private int parentForward = 3;
 
-    transient private CuboidScheduler cuboidScheduler = null;
-
     public boolean isEnableSharding() {
         //in the future may extend to other storage that is shard-able
         return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID;
@@ -459,14 +457,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         return true;
     }
 
-    /**
-     * Get cuboid level count except base cuboid
-     * @return
-     */
-    public int getBuildLevel() {
-        return getCuboidScheduler().getCuboidsByLayer().size() - 1;
-    }
-
     @Override
     public int hashCode() {
         int result = 0;
@@ -560,7 +550,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         hostToDerivedMap = Maps.newHashMap();
         extendedColumnToHosts = Maps.newHashMap();
         cuboidBlackSet = Sets.newHashSet();
-        cuboidScheduler = null;
     }
 
     public void init(KylinConfig config) {
@@ -618,16 +607,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         amendAllColumns();
     }
 
-    public CuboidScheduler getCuboidScheduler() {
-        if (cuboidScheduler != null)
-            return cuboidScheduler;
+    public DefaultCuboidScheduler getInitialCuboidScheduler() {
+        return new DefaultCuboidScheduler(this);
+    }
 
-        synchronized (this) {
-            if (cuboidScheduler == null) {
-                cuboidScheduler = CuboidScheduler.getInstance(this);
-            }
-            return cuboidScheduler;
-        }
+    /**
+     * Get cuboid level count except base cuboid
+     * @return
+     */
+    public int getInitialBuildLevel() {
+        return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1;
     }
 
     public boolean isBlackedCuboid(long cuboidID) {
@@ -1169,7 +1158,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
     }
 
     public Set<Long> getAllCuboids() {
-        return getCuboidScheduler().getAllCuboidIds();
+        return getInitialCuboidScheduler().getAllCuboidIds();
     }
 
     public int getParentForward() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 223df7c..73da802 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -53,8 +53,7 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, Serializ
 
     // check what columns from hive tables are required, and index them
     private void parseCubeDesc() {
-        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
 
         // build index for rowkey columns
         List<TblColRef> cuboidColumns = baseCuboid.getColumns();

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 1c3c395..ec4c04a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -57,7 +57,7 @@ public class CubingUtils {
     public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
         final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
         final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
-        final Set<Long> allCuboidIds = cubeDesc.getCuboidScheduler().getAllCuboidIds();
+        final Set<Long> allCuboidIds = cubeDesc.getInitialCuboidScheduler().getAllCuboidIds();
         final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMap();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 58c0edb..3f3a7c4 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -115,7 +115,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
             System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
 
             for (long child : spannings) {
-                assertTrue(Cuboid.isValid(cube, child));
+                assertTrue(scheduler.isValid(child));
             }
         }
 
@@ -128,7 +128,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testGetSpanningCuboid2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         // generate 8d
         System.out.println("Spanning for 8D Cuboids");
@@ -156,7 +156,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testGetSpanningCuboid1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         // generate 7d
         System.out.println("Spanning for 7D Cuboids");
@@ -228,7 +228,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -241,7 +241,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts2() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -254,7 +254,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts3() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -267,7 +267,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts4() {
         CubeDesc cube = getTestKylinCubeWithSellerLeft();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -280,7 +280,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts5() {
         CubeDesc cube = getStreamingCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -293,7 +293,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts6() {
         CubeDesc cube = getCIInnerJoinCube();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -306,7 +306,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testLargeCube() {
         CubeDesc cube = getFiftyDimCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         long start = System.currentTimeMillis();
         System.out.println(cuboidScheduler.getCuboidCount());
         System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
@@ -317,7 +317,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
         twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
         CubeDesc cube = getTwentyDimCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         cuboidScheduler.getCuboidCount();
         twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
     }
@@ -332,10 +332,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         }
         CubeDescManager.clearCache();
         CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number");
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
-        assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+        assertTrue(scheduler.isValid(baseCuboid.getId()));
 
         List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId());
         assertTrue(spanningChild.size() > 0);

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 2e64791..8dc944b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -73,25 +73,26 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     public void testIsValid() {
 
         CubeDesc cube = getTestKylinCubeWithSeller();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
         // base
-        assertEquals(false, Cuboid.isValid(cube, 0));
-        assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
+        assertEquals(false, cuboidScheduler.isValid(0));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111111111")));
 
         // mandatory column
-        assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("011111110")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("100000000")));
 
         // zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111111000")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("110000111")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("110111000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111110111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("111110001")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("111110100")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("110000100")));
     }
 
     @Test
@@ -124,41 +125,45 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testIsValid2() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
+
+        assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));
 
         // base
-        assertEquals(false, Cuboid.isValid(cube, 0));
-        assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
+        assertEquals(false, cuboidScheduler.isValid(0));
+        assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));
 
         // hierarchy
-        assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
     }
 
     @Test
     public void testIsValid3() {
         CubeDesc cube = getSSBCubeDesc();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
-        assertEquals(false, Cuboid.isValid(cube, toLong("10000000000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10000000000")));
 
         // the 4th is mandatory and isMandatoryOnlyValid is true
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000001000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("00000001000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000001000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("00000001000")));
     }
 
     @Test
     public void testFindCuboidByIdWithSingleAggrGroup2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, 0);
@@ -186,6 +191,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testFindCuboidByIdWithMultiAggrGroup() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, toLong("111111110"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 1d1d147..ec1f221 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -91,7 +91,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
         data[7] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 75e2458..5af8d8a 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -66,7 +66,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[7] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -97,7 +97,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[8] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -133,7 +133,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[8] = null;
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index a3af511..a1a39be 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -131,7 +131,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
         Set<TblColRef> dimensionsD = new LinkedHashSet<TblColRef>();
         dimensionsD.addAll(groupsD);
         dimensionsD.addAll(otherDimsD);
-        Cuboid cuboid = findCuboid(cubeDesc, dimensionsD, metrics);
+        Cuboid cuboid = findCuboid(cubeInstance, dimensionsD, metrics);
         context.setCuboid(cuboid);
 
         // set whether to aggr at storage
@@ -171,8 +171,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
 
     protected abstract String getGTStorage();
 
-    protected Cuboid findCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
-        return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
+    protected Cuboid findCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
+        return Cuboid.identifyCuboid(cubeInstance, dimensionsD, metrics);
     }
 
     protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid,

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index 5db3611..85678ac 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -84,7 +84,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
     public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) {
         this.cubeSeg = cubeSeg;
         long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns);
-        this.cuboid = Cuboid.findById(cubeDesc, cuboidId);
+        this.cuboid = Cuboid.findById(cubeSeg, cuboidId);
         this.flatOrAndFilter = Lists.newLinkedList();
         this.flatOrAndFilter.add(andDimensionRanges);
         init(andDimensionRanges);

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 1ec23b6..f64365a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -64,7 +64,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
 
         // Phase 3: Build Cube
         RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
-        final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
+        final int groupRowkeyColumnsCount = seg.getCuboidScheduler().getBuildLevel();
         // base cuboid step
         result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
         // n dim cuboid steps

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 106077c..51c9056 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -76,7 +76,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
     }
 
     protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
-        final int maxLevel = seg.getCubeDesc().getBuildLevel();
+        final int maxLevel = seg.getCuboidScheduler().getBuildLevel();
         // base cuboid step
         result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
         // n dim cuboid steps

http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
index 07b636b..40f1ac5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -18,7 +18,11 @@
 
 package org.apache.kylin.engine.mr.common;
 
-import com.google.common.collect.Sets;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.CubeSegment;
@@ -35,10 +39,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.collect.Sets;
 
 /**
  */
@@ -88,8 +89,7 @@ public class BaseCuboidBuilder implements java.io.Serializable {
     }
 
     private void init() {
-        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
         initNullBytes();
     }
 


Mime
View raw message