kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [4/4] kylin git commit: KYLIN-2783 Code review clean up
Date Sat, 09 Sep 2017 13:09:38 GMT
KYLIN-2783 Code review clean up


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d28187c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d28187c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d28187c

Branch: refs/heads/pr69
Commit: 4d28187c626768c3813d74acb03b48bd106cc065
Parents: 8eafe27
Author: Li Yang <liyang@apache.org>
Authored: Sat Sep 9 21:09:02 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Sat Sep 9 21:09:02 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeInstance.java     | 47 +++-----------------
 .../java/org/apache/kylin/cube/CubeManager.java |  6 +--
 .../org/apache/kylin/cube/cuboid/Cuboid.java    | 37 +++++++--------
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 14 +++---
 .../kylin/cube/cuboid/CuboidScheduler.java      |  8 ++--
 .../cube/cuboid/DefaultCuboidScheduler.java     |  8 ++--
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  |  2 +-
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  5 ---
 .../kylin/cube/model/AggregationGroup.java      |  2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   | 41 +++++++++--------
 .../apache/kylin/cube/cuboid/CuboidTest.java    |  2 -
 .../apache/kylin/engine/spark/SparkCubing.java  |  4 +-
 12 files changed, 67 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fa270fd..d5d6ac9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,9 +18,6 @@
 
 package org.apache.kylin.cube;
 
-import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT;
-import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND;
-
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
@@ -33,7 +30,6 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.CompressionUtils;
 import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.cuboid.CuboidModeEnum;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -106,14 +102,14 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     @JsonProperty("create_time_utc")
     private long createTimeUTC;
 
-    @JsonProperty("cuboidBytes")
+    @JsonProperty("cuboid_bytes")
     private byte[] cuboidBytes;
 
     @JsonProperty("cuboid_bytes_recommend")
     private byte[] cuboidBytesRecommend;
 
-    @JsonProperty("last_optimized")
-    private long lastOptimized;
+    @JsonProperty("cuboid_last_optimized")
+    private long cuboidLastOptimized;
 
     // cuboid scheduler lazy built
     transient private CuboidScheduler cuboidScheduler;
@@ -332,36 +328,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         this.createTimeUTC = createTimeUTC;
     }
 
-    public Set<Long> getCuboidsByMode(String cuboidModeName) {
-        return getCuboidsByMode(cuboidModeName == null ? null : CuboidModeEnum.getByModeName(cuboidModeName));
-    }
-
-    public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) {
-        if (cuboidMode == null || cuboidMode == CURRENT) {
-            return getCuboidScheduler().getAllCuboidIds();
-        }
-        Set<Long> cuboidsRecommend = getCuboidsRecommend();
-        if (cuboidsRecommend == null || cuboidMode == RECOMMEND) {
-            return cuboidsRecommend;
-        }
-        Set<Long> currentCuboids = getCuboidScheduler().getAllCuboidIds();
-        switch (cuboidMode) {
-        case RECOMMEND_EXISTING:
-            cuboidsRecommend.retainAll(currentCuboids);
-            return cuboidsRecommend;
-        case RECOMMEND_MISSING:
-            cuboidsRecommend.removeAll(currentCuboids);
-            return cuboidsRecommend;
-        case RECOMMEND_MISSING_WITH_BASE:
-            cuboidsRecommend.removeAll(currentCuboids);
-            currentCuboids.add(getCuboidScheduler().getBaseCuboidId());
-            return cuboidsRecommend;
-        default:
-            return null;
-        }
-    }
-
-    public Map<Long, Long> getCuboids() {
+    Map<Long, Long> getCuboids() {
         if (cuboidBytes == null)
             return null;
         byte[] uncompressed;
@@ -377,7 +344,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         }
     }
 
-    public void setCuboids(Map<Long, Long> cuboids) {
+    void setCuboids(Map<Long, Long> cuboids) {
         if (cuboids == null)
             return;
         if (cuboids.isEmpty()) {
@@ -394,7 +361,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         }
     }
 
-    public Set<Long> getCuboidsRecommend() {
+    Set<Long> getCuboidsRecommend() {
         if (cuboidBytesRecommend == null)
             return null;
         byte[] uncompressed;
@@ -410,7 +377,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         }
     }
 
-    public void setCuboidsRecommend(HashSet<Long> cuboids) {
+    void setCuboidsRecommend(HashSet<Long> cuboids) {
         if (cuboids == null)
             return;
         if (cuboids.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6ebfd99..eb330ae 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -310,7 +310,7 @@ public class CubeManager implements IRealizationProvider {
         // remove cube and update cache
         getStore().deleteResource(cube.getResourcePath());
         cubeMap.remove(cube.getName());
-        Cuboid.reloadCache(cube);
+        Cuboid.clearCache(cube);
 
         if (deleteDesc && cube.getDescriptor() != null) {
             CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
@@ -632,7 +632,7 @@ public class CubeManager implements IRealizationProvider {
      */
     public CubeInstance reloadCubeLocal(String cubeName) {
         CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
-        Cuboid.reloadCache(cubeInstance);
+        Cuboid.clearCache(cubeInstance);
         return cubeInstance;
     }
 
@@ -643,7 +643,7 @@ public class CubeManager implements IRealizationProvider {
             for (CubeSegment segment : cube.getSegments()) {
                 usedStorageLocation.remove(segment.getUuid());
             }
-            Cuboid.reloadCache(cube);
+            Cuboid.clearCache(cube);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index fa83775..f314baf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -45,7 +45,7 @@ import com.google.common.collect.Maps;
 @SuppressWarnings("serial")
 public class Cuboid implements Comparable<Cuboid>, Serializable {
 
-    //TODO Guava cache may be better
+    // TODO Should the cache be inside CuboidScheduler?
     private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap();
 
     // smaller is better
@@ -88,11 +88,6 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         return cuboidID;
     }
 
-    // for mandatory cuboid, no need to translate cuboid
-    public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) {
-        return new Cuboid(cube, cuboidID, cuboidID);
-    }
-
     public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) {
         return findById(cuboidScheduler, Bytes.toLong(cuboidID));
     }
@@ -111,10 +106,10 @@ public class Cuboid implements Comparable<Cuboid>, Serializable
{
     }
 
     public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) {
-        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey());
+        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getCuboidCacheKey());
         if (cubeCache == null) {
             cubeCache = Maps.newConcurrentMap();
-            CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache);
+            CUBOID_CACHE.put(cuboidScheduler.getCuboidCacheKey(), cubeCache);
         }
         Cuboid cuboid = cubeCache.get(cuboidID);
         if (cuboid == null) {
@@ -130,7 +125,19 @@ public class Cuboid implements Comparable<Cuboid>, Serializable
{
     }
 
     public static Cuboid getBaseCuboid(CubeDesc cube) {
-        return findForMandatory(cube, getBaseCuboidId(cube));
+        return findById(cube.getInitialCuboidScheduler(), getBaseCuboidId(cube));
+    }
+
+    public static void clearCache() {
+        CUBOID_CACHE.clear();
+    }
+
+    public static void clearCache(CubeInstance cubeInstance) {
+        clearCache(cubeInstance.getCuboidScheduler());
+    }
+
+    private static void clearCache(CuboidScheduler cuboidScheduler) {
+        CUBOID_CACHE.remove(cuboidScheduler.getCuboidCacheKey());
     }
 
     // ============================================================================
@@ -226,18 +233,6 @@ public class Cuboid implements Comparable<Cuboid>, Serializable
{
         return requirePostAggregation;
     }
 
-    public static void clearCache() {
-        CUBOID_CACHE.clear();
-    }
-
-    public static void reloadCache(CubeInstance cubeInstance) {
-        reloadCache(cubeInstance.getCuboidScheduler());
-    }
-
-    private static void reloadCache(CuboidScheduler cuboidScheduler) {
-        CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey());
-    }
-
     @Override
     public String toString() {
         return "Cuboid [id=" + id + "]";

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 4300a6c..fa8e879 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -88,28 +88,28 @@ public class CuboidCLI {
                 //check all valid and invalid
                 for (long i = 0; i < baseCuboid; ++i) {
                     if (cuboidSet.contains(i)) {
-                        if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) {
+                        if (!scheduler.isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) !=
i) {
+                        if (scheduler.findBestMatchCuboid(i) != i) {
                             throw new RuntimeException();
                         }
                     } else {
-                        if (cubeDesc.getInitialCuboidScheduler().isValid(i)) {
+                        if (scheduler.isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i);
+                        long corrected = scheduler.findBestMatchCuboid(i);
                         if (corrected == i) {
                             throw new RuntimeException();
                         }
 
-                        if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) {
+                        if (!scheduler.isValid(corrected)) {
                             throw new RuntimeException();
                         }
 
-                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected)
!= corrected) {
+                        if (scheduler.findBestMatchCuboid(corrected) != corrected) {
                             throw new RuntimeException();
                         }
                     }
@@ -133,7 +133,7 @@ public class CuboidCLI {
     }
 
     public static int[] calculateAllLevelCount(CubeDesc cube) {
-        int levels = cube.getInitialBuildLevel();
+        int levels = cube.getInitialCuboidScheduler().getBuildLevel();
         int[] allLevelCounts = new int[levels + 1];
 
         CuboidScheduler scheduler = cube.getInitialCuboidScheduler();

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index cef0f77..5de7e9e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -60,14 +60,14 @@ abstract public class CuboidScheduler {
     /** Returns the child cuboids of a parent. */
     abstract public List<Long> getSpanningCuboid(long parentCuboid);
     
-    /** Returns a cuboid on the tree that best matches the request cuboid. */
+    /** Returns a valid cuboid that best matches the request cuboid. */
     abstract public long findBestMatchCuboid(long requestCuboid);
 
-    /** Returns whether requestCuboid is valid or not*/
+    /** Checks whether a cuboid is valid or not. */
     abstract public boolean isValid(long requestCuboid);
 
-    /** Returns the key for what this cuboid scheduler responsible for*/
-    abstract public String getResponsibleKey();
+    /** Returns the key for what this cuboid scheduler responsible for. */
+    abstract public String getCuboidCacheKey();
 
     // ============================================================================
     

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index cba417a..9762cdc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -102,7 +102,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return findBestMatchCuboid1(cuboid);
     }
 
-    public long findBestMatchCuboid1(long cuboid) {
+    long findBestMatchCuboid1(long cuboid) {
         if (isValid(cuboid)) {
             return cuboid;
         }
@@ -127,7 +127,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return doFindBestMatchCuboid1(onTreeCandi);
     }
 
-    public long doFindBestMatchCuboid1(long cuboid) {
+    private long doFindBestMatchCuboid1(long cuboid) {
         long parent = getOnTreeParent(cuboid);
         while (parent > 0) {
             if (cubeDesc.getAllCuboids().contains(parent)) {
@@ -466,7 +466,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return Long.bitCount(cuboidID) <= dimCap;
     }
 
-    public long findBestMatchCuboid2(long cuboid) {
+    long findBestMatchCuboid2(long cuboid) {
         long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc));
         if (bestParent < -1) {
             throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid);
@@ -499,7 +499,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return (cuboidId & ~parentCuboid) == 0;
     }
 
-    public String getResponsibleKey() {
+    public String getCuboidCacheKey() {
         return CubeDesc.class.getName() + "-" + cubeDesc.getName();
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 414fc4a..e0e82aa 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -316,7 +316,7 @@ public class TreeCuboidScheduler extends CuboidScheduler {
         }
     }
 
-    public String getResponsibleKey() {
+    public String getCuboidCacheKey() {
         return CubeInstance.class.getName() + "-" + cubeDesc.getName();
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index ae38261..952926c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -51,11 +51,6 @@ abstract public class AbstractInMemCubeBuilder {
     protected int taskThreadCount = 1;
     protected int reserveMemoryMB = 100;
 
-    //    @Deprecated
-    //    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc,
Map<TblColRef, Dictionary<String>> dictionaryMap) {
-    //        this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap);
-    //    }
-
     protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc
flatDesc,
             Map<TblColRef, Dictionary<String>> dictionaryMap) {
         if (cuboidScheduler == null)

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 852451d..c1dbc1e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -296,7 +296,7 @@ public class AggregationGroup implements Serializable {
         long combination = 1;
 
         if (this.getDimCap() > 0) {
-            DefaultCuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+            DefaultCuboidScheduler cuboidScheduler = (DefaultCuboidScheduler) cubeDesc.getInitialCuboidScheduler();
             combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
         } else {
             Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index dff3485..f0b14cc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -51,7 +51,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
@@ -168,15 +168,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
     @JsonProperty("override_kylin_properties")
     private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String,
String>();
 
-    private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
-    private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
-    private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
-
-    private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
-    private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap =
Maps.newHashMap();
-
-    private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap();
-
     @JsonProperty("partition_offset_start")
     @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private Map<Integer, Long> partitionOffsetStart = Maps.newHashMap();
@@ -189,6 +180,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
     @JsonInclude(JsonInclude.Include.NON_NULL)
     private int parentForward = 3;
 
+    private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
+    private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
+    private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
+
+    private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
+    private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap =
Maps.newHashMap();
+
+    private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap();
+    
+    transient private CuboidScheduler cuboidScheduler = null;
+
     public boolean isEnableSharding() {
         //in the future may extend to other storage that is shard-able
         return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID;
@@ -550,6 +552,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
         hostToDerivedMap = Maps.newHashMap();
         extendedColumnToHosts = Maps.newHashMap();
         cuboidBlackSet = Sets.newHashSet();
+        cuboidScheduler = null;
     }
 
     public void init(KylinConfig config) {
@@ -607,16 +610,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
         amendAllColumns();
     }
 
-    public DefaultCuboidScheduler getInitialCuboidScheduler() {
-        return new DefaultCuboidScheduler(this);
-    }
+    public CuboidScheduler getInitialCuboidScheduler() {
+        if (cuboidScheduler != null)
+            return cuboidScheduler;
 
-    /**
-     * Get cuboid level count except base cuboid
-     * @return
-     */
-    public int getInitialBuildLevel() {
-        return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1;
+        synchronized (this) {
+            if (cuboidScheduler == null) {
+                cuboidScheduler = CuboidScheduler.getInstance(this);
+            }
+            return cuboidScheduler;
+        }
     }
 
     public boolean isBlackedCuboid(long cuboidID) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 8dc944b..0fa849e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -163,7 +163,6 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testFindCuboidByIdWithSingleAggrGroup2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, 0);
@@ -191,7 +190,6 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testFindCuboidByIdWithMultiAggrGroup() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
-        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, toLong("111111110"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d28187c/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index fd53c5a..7205e1b 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -377,8 +377,8 @@ public class SparkCubing extends AbstractApplication {
                 LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue();
                 System.out.println("load properties finished");
                 IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
-                        AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(
-                                cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap);
+                AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(
+                        cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap);
                 final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new
DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap));
                 Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue,
sparkCuboidWriter));
                 try {


Mime
View raw message