kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [4/7] incubator-kylin git commit: whitelist
Date Mon, 30 Nov 2015 03:16:25 GMT
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
new file mode 100644
index 0000000..4e7b99e
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ *  find forbid overlaps in each AggregationGroup
+ *  the include dims in AggregationGroup must contain all mandatory, hierarchy and joint
+ */
+public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
+     * , org.apache.kylin.metadata.validation.ValidateContext)
+     */
+    @Override
+    public void validate(CubeDesc cube, ValidateContext context) {
+        innerValidateMaxSize(cube, context);
+    }
+
+    /**
+     * @param cube
+     * @param context
+     */
+    private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
+        int maxSize = getMaxAgrGroupSize();
+
+        int index = 0;
+        for (AggregationGroup agg : cube.getAggregationGroups()) {
+            if (agg.getIncludes() == null) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " include dims not set");
+                continue;
+            }
+
+            if (agg.getSelectRule() == null) {
+                continue;
+            }
+
+            Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getIncludes() != null) {
+                for (String include : agg.getIncludes()) {
+                    includeDims.add(include);
+                }
+            }
+
+            Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().mandatory_dims != null) {
+                for (String include : agg.getSelectRule().mandatory_dims) {
+                    mandatoryDims.add(include);
+                }
+            }
+
+            Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().hierarchy_dims != null) {
+                for (String[] ss : agg.getSelectRule().hierarchy_dims) {
+                    for (String s : ss)
+                        hierarchyDims.add(s);
+                }
+            }
+
+            Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().joint_dims != null) {
+                for (String[] ss : agg.getSelectRule().joint_dims) {
+                    for (String s : ss)
+                        jointDims.add(s);
+                }
+            }
+
+            if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " Include dims not containing all the used dims");
+            }
+
+            Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            normalDims.addAll(includeDims);
+            normalDims.removeAll(mandatoryDims);
+            normalDims.removeAll(hierarchyDims);
+            normalDims.removeAll(jointDims);
+
+            int normalDimSize = normalDims.size();
+            int hierarchySize = (agg.getSelectRule().hierarchy_dims == null ? 0 : agg.getSelectRule().hierarchy_dims.length);
+            int jointSize = agg.getSelectRule().joint_dims == null ? 0 : agg.getSelectRule().joint_dims.length;
+            if (normalDimSize + hierarchySize + jointSize > maxSize) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions");
+            }
+
+            if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+            }
+            if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims");
+            }
+
+            int jointDimNum = 0;
+            if (agg.getSelectRule().joint_dims != null) {
+                for (String[] joints : agg.getSelectRule().joint_dims) {
+
+                    Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+                    for (String s : joints) {
+                        oneJoint.add(s);
+                    }
+
+                    if (oneJoint.size() < 2) {
+                        context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint");
+                    }
+                    jointDimNum += oneJoint.size();
+
+                    int overlapHierarchies = 0;
+                    if (agg.getSelectRule().hierarchy_dims != null) {
+                        for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) {
+                            Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+                            share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy)));
+
+                            if (!share.isEmpty()) {
+                                overlapHierarchies++;
+                            }
+                            if (share.size() > 1) {
+                                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy");
+                            }
+                        }
+
+                        if (overlapHierarchies > 1) {
+                            context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies");
+                        }
+                    }
+                }
+
+                if (jointDimNum != jointDims.size()) {
+                    context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint");
+                }
+            }
+
+            index++;
+        }
+    }
+
+    protected int getMaxAgrGroupSize() {
+        String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
+        return Integer.parseInt(size);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index e37b9a9..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- * 
- * @author jianliu
- * 
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
-     * , org.apache.kylin.metadata.validation.ValidateContext)
-     */
-    @Override
-    public void validate(CubeDesc cube, ValidateContext context) {
-        innerValidateMaxSize(cube, context);
-    }
-
-    /**
-     * @param cube
-     * @param context
-     */
-    private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
-        int maxSize = getMaxAgrGroupSize();
-        String[][] groups = cube.getRowkey().getAggregationGroups();
-        for (int i = 0; i < groups.length; i++) {
-            String[] group = groups[i];
-            if (group.length >= maxSize) {
-                context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
-            }
-        }
-    }
-
-    protected int getMaxAgrGroupSize() {
-        String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
-        return Integer.parseInt(size);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
index 52e5b24..80d4676 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
@@ -20,13 +20,9 @@ package org.apache.kylin.cube.model.validation.rule;
 
 import org.apache.kylin.metadata.MetadataConstants;
 
-/**
- * @author jianliu
- * 
- */
 public interface IKylinValidationConstants extends MetadataConstants {
 
-    public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
+    public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 12;
     public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
     public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index 4585900..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- * 
- * @author jianliu
- * 
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
-     * , org.apache.kylin.metadata.validation.ValidateContext)
-     */
-    @Override
-    public void validate(CubeDesc cube, ValidateContext context) {
-        Set<String> mands = new HashSet<String>();
-        RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
-        if (cols == null || cols.length == 0) {
-            return;
-        }
-        for (int i = 0; i < cols.length; i++) {
-            RowKeyColDesc rowKeyColDesc = cols[i];
-            if (rowKeyColDesc.isMandatory()) {
-                mands.add(rowKeyColDesc.getColumn());
-            }
-        }
-        if (mands.isEmpty()) {
-            return;
-        }
-        String[][] groups = cube.getRowkey().getAggregationGroups();
-        for (int i = 0; i < groups.length; i++) {
-            String[] group = groups[i];
-            for (int j = 0; j < group.length; j++) {
-                String col = group[j];
-                if (mands.contains(col)) {
-                    context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
-                }
-            }
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
index e90f6cd..7f4b6e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.cube.model.validation.rule;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.cube.model.RowKeyDesc;
@@ -28,9 +27,6 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
 
 /**
  * Validate that only one of "length" and "dictionary" appears on rowkey_column
- * 
- * @author jianliu
- * 
  */
 public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
 
@@ -61,12 +57,12 @@ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
 
         for (int i = 0; i < rcd.length; i++) {
             RowKeyColDesc rd = rcd[i];
-            if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
-                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
-            }
-            if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
-                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
-            }
+            //            if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
+            //                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
+            //            }
+            //            if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
+            //                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
+            //            }
         }
 
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
new file mode 100644
index 0000000..772da17
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.cube.model.v2.CubeDesc;
+import org.apache.kylin.cube.model.v2.DimensionDesc;
+import org.apache.kylin.cube.model.v2.HBaseMappingDesc;
+import org.apache.kylin.cube.model.v2.RowKeyColDesc;
+import org.apache.kylin.cube.model.v2.RowKeyDesc;
+
+import com.google.common.collect.Lists;
+
+public class CubeDescUpgraderV2 {
+
+    @SuppressWarnings("unused")
+    private static final Log logger = LogFactory.getLog(CubeDescUpgraderV2.class);
+    private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
+
+    private String resourcePath;
+
+    private List<String[]> oldHierarchies = Lists.newArrayList();
+    private List<String> oldMandatories = Lists.newArrayList();
+    private String[][] oldAggGroup = null;
+    private Set<String> allRowKeyCols = newIgnoreCaseSet(null);
+
+    public CubeDescUpgraderV2(String resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
+        CubeDesc oldModel = loadOldCubeDesc(resourcePath);
+
+        org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
+        copyUnChangedProperties(oldModel, newModel);
+        upgradeDimension(oldModel, newModel);
+        upgradeRowKey(oldModel, newModel);
+        upgradeHBaseMapping(oldModel, newModel);
+        upgradeAggregationGroup(newModel);//must do at last
+
+        return newModel;
+    }
+
+    private CubeDesc loadOldCubeDesc(String path) throws IOException {
+        ResourceStore store = getStore();
+
+        CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer);
+
+        if (StringUtils.isBlank(ndesc.getName())) {
+            throw new IllegalStateException("CubeDesc name must not be blank");
+        }
+
+        return ndesc;
+    }
+
+    private Set<String> newIgnoreCaseSet(Set<String> input) {
+        Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+        if (input != null)
+            ret.addAll(input);
+        return ret;
+    }
+
+    private String[] toArray(Set<String> input) {
+        return input.toArray(new String[input.size()]);
+    }
+
+    private boolean rowKeyColExistsInMultipleAggGroup() {
+        if (oldAggGroup == null)
+            return false;
+
+        int total = 0;
+        Set<String> overall = newIgnoreCaseSet(null);
+        for (String[] group : oldAggGroup) {
+            Set<String> temp = newIgnoreCaseSet(null);
+            for (String entry : group) {
+
+                overall.add(entry);
+                temp.add(entry);
+            }
+            total += temp.size();
+        }
+        return overall.size() != total;
+    }
+
+    private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) {
+
+        List<AggregationGroup> aggs = Lists.newArrayList();
+        if (oldAggGroup == null || oldAggGroup.length == 0 || //
+                ((oldAggGroup.length == 1) && (oldAggGroup[0].length == allRowKeyCols.size()))) {
+            AggregationGroup agg = new AggregationGroup();
+            agg.setIncludes(toArray(allRowKeyCols));
+            agg.setSelectRule(null);
+            aggs.add(agg);
+        } else {
+            if (rowKeyColExistsInMultipleAggGroup()) {
+                throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!");
+            }
+
+            Set<String> visited = newIgnoreCaseSet(null);
+
+            for (String[] group : oldAggGroup) {
+                AggregationGroup agg = new AggregationGroup();
+
+                Set<String> remaining = newIgnoreCaseSet(allRowKeyCols);
+                remaining.removeAll(visited);
+
+                Set<String> joint = newIgnoreCaseSet(remaining);
+                joint.removeAll(oldMandatories);
+
+                Set<String> groupAsSet = newIgnoreCaseSet(null);
+                for (String entry : group) {
+                    groupAsSet.add(entry);
+                }
+                visited.addAll(groupAsSet);
+                joint.removeAll(groupAsSet);
+
+                List<String> mandatories = Lists.newArrayList();
+                List<String[]> hierarchies = Lists.newArrayList();
+
+                for (String s : oldMandatories) {
+                    mandatories.add(s);
+                }
+
+                for (String[] h : oldHierarchies) {
+                    if (groupAsSet.containsAll(Arrays.asList(h))) {
+                        hierarchies.add(h);
+                    }
+                }
+
+                agg.setIncludes(toArray(remaining));
+
+                SelectRule selectRule = new SelectRule();
+                selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]);
+                if (joint.size() != 0) {
+                    selectRule.joint_dims = new String[1][];
+                    selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]);
+                } else {
+                    selectRule.joint_dims = new String[0][];
+                }
+                selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]);
+                agg.setSelectRule(selectRule);
+
+                aggs.add(agg);
+
+            }
+        }
+        newModel.setAggregationGroups(aggs);
+    }
+
+    private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        List<DimensionDesc> oldDimensions = oldModel.getDimensions();
+        if (oldDimensions == null) {
+            throw new IllegalArgumentException("dimensions is null");
+        }
+        List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList();
+
+        for (DimensionDesc oldDim : oldDimensions) {
+            if (oldDim.isDerived()) {
+                org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+                newDim.setName(oldDim.getName());
+                newDim.setTable(oldDim.getTable());
+                newDim.setColumn("{FK}");
+                newDim.setDerived(oldDim.getDerived());
+
+                newDimensions.add(newDim);
+            } else {
+                if (oldDim.isHierarchy()) {
+                    oldHierarchies.add(oldDim.getColumn());
+                }
+
+                for (String columnStr : oldDim.getColumn()) {
+                    org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+                    newDim.setName(oldDim.getName());
+                    newDim.setTable(oldDim.getTable());
+                    newDim.setColumn(columnStr);
+                    newDim.setDerived(null);
+
+                    newDimensions.add(newDim);
+                }
+            }
+        }
+
+        newModel.setDimensions(newDimensions);
+    }
+
+    private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        RowKeyDesc oldRowKey = oldModel.getRowkey();
+        if (oldModel == null) {
+            throw new IllegalArgumentException("RowKeyDesc is null");
+        }
+
+        if (oldRowKey.getRowKeyColumns() == null) {
+            throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null");
+        }
+
+        org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc();
+        org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length];
+        int index = 0;
+        for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) {
+            org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc();
+
+            allRowKeyCols.add(oldRowKeyCol.getColumn());
+            if (oldRowKeyCol.isMandatory()) {
+                oldMandatories.add(oldRowKeyCol.getColumn());
+            }
+
+            newRowKeyCol.setColumn(oldRowKeyCol.getColumn());
+            if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) {
+                newRowKeyCol.setEncoding("dict");
+            } else if (oldRowKeyCol.getLength() > 0) {
+                newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength());
+            } else {
+                throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength());
+            }
+            cols[index++] = newRowKeyCol;
+        }
+        oldAggGroup = oldRowKey.getAggregationGroups();
+
+        newRowKey.setRowkeyColumns(cols);
+        newModel.setRowkey(newRowKey);
+    }
+
+    private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping();
+        try {
+
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            JsonUtil.writeValueIndent(os, hbaseMappingDesc);
+            byte[] blob = os.toByteArray();
+            ByteArrayInputStream is = new ByteArrayInputStream(blob);
+            org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class);
+            newModel.setHbaseMapping(newHBaseMappingDesc);
+
+        } catch (IOException e) {
+            throw new RuntimeException("error when copying HBaseMappingDesc");
+        }
+    }
+
+    private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        newModel.setUuid(oldModel.getUuid());
+        newModel.setName(oldModel.getName());
+        newModel.setDescription(oldModel.getDescription());
+        newModel.setMeasures(oldModel.getMeasures());
+        newModel.setNullStrings(oldModel.getNullStrings());
+        newModel.setModelName(oldModel.getModelName());
+        newModel.setNotifyList(oldModel.getNotifyList());
+        newModel.setLastModified(oldModel.getLastModified());
+        newModel.setStorageType(oldModel.getStorageType());
+        newModel.setEngineType(oldModel.getEngineType());
+    }
+
+    protected static ResourceStore getStore() {
+        return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
new file mode 100644
index 0000000..4d3fce9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+
+/**
+ * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1,
+ * that upgrades metadata store from v1(prior kylin 0.7) to v2.
+ * the major difference is that we split cube desc to cube desc + model desc
+ * 
+ * this CubeMetadataUpgradeV2 upgrades metadata store from v2(prior kylin 2.1) to v3
+ * the major different is a brand new definition of partial cubes to allow users to select 
+ * cuboids more flexibly 
+ */
+public class CubeMetadataUpgradeV2 {
+
+    private KylinConfig config = null;
+    private ResourceStore store;
+
+    private List<String> updatedResources = Lists.newArrayList();
+    private List<String> errorMsgs = Lists.newArrayList();
+
+    private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV2.class);
+
+    public CubeMetadataUpgradeV2(String newMetadataUrl) {
+        KylinConfig.destoryInstance();
+        System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl);
+        KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl);
+
+        config = KylinConfig.getInstanceFromEnv();
+        store = getStore();
+    }
+
+    public void upgrade() {
+
+        upgradeCubeDesc();
+        verify();
+    }
+
+    public void verify() {
+        MetadataManager.clearCache();
+        MetadataManager.getInstance(config);
+        CubeDescManager.clearCache();
+        CubeDescManager.getInstance(config);
+        CubeManager.clearCache();
+        CubeManager.getInstance(config);
+        ProjectManager.clearCache();
+        ProjectManager.getInstance(config);
+        //cleanup();
+    }
+
+    private List<String> listResourceStore(String pathRoot) {
+        List<String> paths = null;
+        try {
+            paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX);
+        } catch (IOException e1) {
+            e1.printStackTrace();
+            errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+        }
+
+        return paths;
+    }
+
+    private void upgradeCubeDesc() {
+        logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+
+        List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+        for (String path : paths) {
+
+            try {
+                CubeDescUpgraderV2 upgrade = new CubeDescUpgraderV2(path);
+                CubeDesc ndesc = upgrade.upgrade();
+                ndesc.setSignature(ndesc.calculateSignature());
+
+                getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+                updatedResources.add(ndesc.getResourcePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+                errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
+            }
+        }
+
+    }
+
+    private ResourceStore getStore() {
+        return ResourceStore.getStore(config);
+    }
+
+    public static void main(String[] args) {
+
+        if (!(args != null && (args.length == 1 || args.length == 2))) {
+            System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
+            return;
+        }
+
+        String exportFolder = args[0];
+        boolean verify = false;
+        if (args.length == 2 && "verify".equals(args[1])) {
+            System.out.println("Only verify the metadata in folder " + exportFolder);
+            verify = true;
+        }
+
+        CubeMetadataUpgradeV2 instance = null;
+        if (verify) {
+            instance = new CubeMetadataUpgradeV2(exportFolder);
+            instance.verify();
+        } else {
+            File oldMetaFolder = new File(exportFolder);
+            if (!oldMetaFolder.exists()) {
+                System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
+                return;
+            }
+
+            if (!oldMetaFolder.isDirectory()) {
+                System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
+                return;
+            }
+
+            String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v3";//upgrades metadata store to v3 format
+            try {
+                FileUtils.deleteDirectory(new File(newMetadataUrl));
+                FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+            instance = new CubeMetadataUpgradeV2(newMetadataUrl);
+            instance.upgrade();
+            logger.info("=================================================================");
+            logger.info("Run CubeMetadataUpgrade completed;");
+
+        }
+
+        logger.info("=================================================================");
+        if (instance.errorMsgs.size() > 0) {
+            logger.info("Here are the error/warning messages, you may need check:");
+            for (String s : instance.errorMsgs) {
+                logger.warn(s);
+            }
+        } else {
+            logger.info("No error or warning messages; The migration is success.");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 0cfd020..ada31b6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -34,7 +34,6 @@
 package org.apache.kylin.cube.util;
 
 import java.io.IOException;
-import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -42,14 +41,6 @@ import java.util.Map;
 
 import javax.annotation.Nullable;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
 import org.apache.kylin.common.util.ByteArray;
@@ -59,12 +50,25 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.source.ReadableTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
 /**
  */
 public class CubingUtils {
@@ -82,8 +86,7 @@ public class CubingUtils {
             @Nullable
             @Override
             public Integer[] apply(@Nullable Long cuboidId) {
-                BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-                Integer[] result = new Integer[bitSet.cardinality()];
+                Integer[] result = new Integer[Long.bitCount(cuboidId)];
 
                 long mask = Long.highestOneBit(baseCuboidId);
                 int position = 0;
@@ -100,8 +103,7 @@ public class CubingUtils {
         final Map<Long, HyperLogLogPlusCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
         for (Long cuboidId : allCuboidIds) {
             result.put(cuboidId, new HyperLogLogPlusCounter(14));
-            BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-            Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+            Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
 
             long mask = Long.highestOneBit(baseCuboidId);
             int position = 0;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
new file mode 100644
index 0000000..6cfaaa4
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author jianliu
+ * 
+ */
+public class AggregationGroupRuleTest {
+
+    private CubeDesc cube;
+    private ValidateContext vContext = new ValidateContext();
+
+    /**
+     * @throws java.lang.Exception
+     */
+    @Before
+    public void setUp() throws Exception {
+        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
+        this.cube = desc2;
+
+    }
+
+    @Test
+    public void testOneMandatoryColumn() {
+        IValidatorRule<CubeDesc> rule = new AggregationGroupRule() {
+            /*
+             * (non-Javadoc)
+             * 
+             * @see
+             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+             * #getMaxAgrGroupSize()
+             */
+            @Override
+            protected int getMaxAgrGroupSize() {
+                return 3;
+            }
+        };
+        rule.validate(cube, vContext);
+        vContext.print(System.out);
+        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
+        assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
+        assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
+        // assertTrue("Failed to validate aggragation group error",
+        // vContext.getResults()[2].getMessage()
+        // .startsWith("Hierachy column"));
+    }
+
+    @Test
+    public void testAggColumnSize() {
+        AggregationGroupRule rule = new AggregationGroupRule() {
+            /*
+             * (non-Javadoc)
+             * 
+             * @see
+             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+             * #getMaxAgrGroupSize()
+             */
+            @Override
+            protected int getMaxAgrGroupSize() {
+                return 20;
+            }
+        };
+        rule.validate(cube, vContext);
+        vContext.print(System.out);
+        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
+        // assertTrue("Failed to validate aggragation group error",
+        // vContext.getResults()[0].getMessage()
+        // .startsWith("Aggregation group"));
+        // assertTrue("Failed to validate aggragation group error",
+        // vContext.getResults()[0].getMessage()
+        // .startsWith("Hierachy column"));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index e7914c7..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- * 
- */
-public class AggregationGroupSizeRuleTest {
-
-    private CubeDesc cube;
-    private ValidateContext vContext = new ValidateContext();
-
-    /**
-     * @throws java.lang.Exception
-     */
-    @Before
-    public void setUp() throws Exception {
-        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
-        this.cube = desc2;
-
-    }
-
-    @Test
-    public void testOneMandatoryColumn() {
-        IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
-            /*
-             * (non-Javadoc)
-             * 
-             * @see
-             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
-             * #getMaxAgrGroupSize()
-             */
-            @Override
-            protected int getMaxAgrGroupSize() {
-                return 3;
-            }
-        };
-        rule.validate(cube, vContext);
-        vContext.print(System.out);
-        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
-        assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
-        assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[2].getMessage()
-        // .startsWith("Hierachy column"));
-    }
-
-    @Test
-    public void testAggColumnSize() {
-        AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
-            /*
-             * (non-Javadoc)
-             * 
-             * @see
-             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
-             * #getMaxAgrGroupSize()
-             */
-            @Override
-            protected int getMaxAgrGroupSize() {
-                return 20;
-            }
-        };
-        rule.validate(cube, vContext);
-        vContext.print(System.out);
-        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[0].getMessage()
-        // .startsWith("Aggregation group"));
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[0].getMessage()
-        // .startsWith("Hierachy column"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index d7feb56..0de5c22 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -56,10 +56,10 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
         CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
         TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
 
-        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
         System.out.println(JsonUtil.writeValueAsIndentString(info1));
 
-        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
         System.out.println(JsonUtil.writeValueAsIndentString(info2));
 
         assertTrue(info1.getUuid() == info2.getUuid());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
deleted file mode 100644
index 29b7183..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- * 
- */
-public class MandatoryColumnRuleTest {
-
-    private CubeDesc cube;
-    private ValidateContext vContext = new ValidateContext();
-
-    /**
-     * @throws java.lang.Exception
-     */
-    @Before
-    public void setUp() throws Exception {
-        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
-        this.cube = desc2;
-
-    }
-
-    @Test
-    public void testOneMandatoryColumn() {
-        IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
-        rule.validate(cube, vContext);
-        assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
-        assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 83e8d94..b45a005 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -83,34 +83,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
     }
 
-    @Test
-    public void testFindSmallerSibling1() {
-        CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler scheduler = new CuboidScheduler(cube);
-
-        Collection<Long> siblings;
-
-        siblings = scheduler.findSmallerSibling(255);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(133);
-        assertEquals("[131]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(127);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(134);
-        assertEquals("[131, 133]", sortToString(siblings));
-
-        siblings = scheduler.findSmallerSibling(130);
-        assertEquals("[129]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(5);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(135);
-        assertEquals("[]", siblings.toString());
-    }
 
     private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
         for (long cuboidId : cuboidIds) {
@@ -123,31 +95,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         }
     }
 
-    @Test
-    public void testFindSmallerSibling2() {
-        CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler scheduler = new CuboidScheduler(cube);
-
-        Collection<Long> siblings;
-
-        siblings = scheduler.findSmallerSibling(511);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("110111111"));
-        assertEquals("[383]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("101110111"));
-        assertEquals("[319]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("111111000"));
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("111111000"));
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("110000000"));
-        assertEquals("[288, 320]", sortToString(siblings));
-    }
 
     @Test
     public void testGetSpanningCuboid2() {
@@ -234,19 +181,19 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidGeneration1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
     public void testCuboidGeneration2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
     public void testCuboidGeneration3() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
@@ -266,6 +213,11 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
     }
 
+    @Test
+    public void testPotentialChild() {
+        assertEquals(1, 2);
+    }
+
     private String sortToString(Collection<Long> longs) {
         ArrayList<Long> copy = new ArrayList<Long>(longs);
         Collections.sort(copy);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
index e37aabc..774ad09 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
@@ -100,7 +100,7 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
         Assert.assertTrue(dims.size() > 0);
 
         for (DimensionDesc dim : dims) {
-            Assert.assertTrue(dim.getColumn().length > 0);
+            Assert.assertTrue(dim.getColumn() != null);
         }
 
         Assert.assertTrue(cubedesc1.getMeasures().size() > 0);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..2a1304a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -223,11 +223,11 @@ public class DictionaryManager {
         }
     }
 
-    public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+    public DictionaryInfo buildDictionary(DataModelDesc model, boolean usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
 
         logger.info("building dictionary for " + col);
 
-        TblColRef srcCol = decideSourceData(model, dict, col);
+        TblColRef srcCol = decideSourceData(model, usingDict, col);
         String srcTable = srcCol.getTable();
         String srcColName = srcCol.getName();
         int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
@@ -259,9 +259,9 @@ public class DictionaryManager {
     /**
      * Decide a dictionary's source data, leverage PK-FK relationship.
      */
-    public TblColRef decideSourceData(DataModelDesc model, String dict, TblColRef col) throws IOException {
+    public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, TblColRef col) throws IOException {
         // Note FK on fact table is supported by scan the related PK on lookup table
-        if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) {
+        if (usingDict) {
             // FK on fact table and join type is inner, use PK from lookup instead
             if (model.isFactTable(col.getTable())) {
                 TblColRef pkCol = model.findPKByFK(col, "inner");
@@ -270,7 +270,7 @@ public class DictionaryManager {
             }
             return col;
         } else
-            throw new IllegalArgumentException("Unknown dictionary value: " + dict);
+            throw new IllegalArgumentException("Not using Dictionary ");
     }
 
     private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 1561b1f..de471c1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -28,8 +28,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
 public class MeasureDesc {
 
-    @JsonProperty("id")
-    private int id;
     @JsonProperty("name")
     private String name;
     @JsonProperty("function")
@@ -37,14 +35,6 @@ public class MeasureDesc {
     @JsonProperty("dependent_measure_ref")
     private String dependentMeasureRef;
 
-    public int getId() {
-        return id;
-    }
-
-    public void setId(int id) {
-        this.id = id;
-    }
-
     public String getName() {
         return name;
     }
@@ -91,11 +81,6 @@ public class MeasureDesc {
     }
 
     @Override
-    public int hashCode() {
-        return id;
-    }
-
-    @Override
     public String toString() {
         return "MeasureDesc [name=" + name + ", function=" + function + "]";
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 12cac94..3c10c09 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -63,7 +63,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
 
         // Phase 3: Build Cube
         RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
-        final int groupRowkeyColumnsCount = rowKeyDesc.getNCuboidBuildLevels();
+        final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
         final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
         final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
         // base cuboid step

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index b97c88a..b656da2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -62,7 +62,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
         for (int i = 0; i < dictionaryColumns.size(); i++) {
             TblColRef col = dictionaryColumns.get(i);
 
-            String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+            String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
             if (cubeDesc.getModel().isFactTable(scanTable)) {
                 factDictCols.add(i);
             }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index e43d5d1..0c5b9b3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -89,8 +89,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
 
     private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) {
         allCuboids.add(cuboidId);
-        BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-        Integer[] indice = new Integer[bitSet.cardinality()];
+        Integer[] indice = new Integer[Long.bitCount(cuboidId)];
 
         long mask = Long.highestOneBit(baseCuboidId);
         int position = 0;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index 6098381..8416d95 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -88,7 +88,7 @@ public class MapContextGTRecordWriter implements ICuboidWriter {
         rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
         keyBuf = rowKeyEncoder.createBuf();
 
-        dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+        dimensions = Long.bitCount(cuboidId);
         measureColumnsIndex = new int[measureCount];
         for (int i = 0; i < measureCount; i++) {
             measureColumnsIndex[i] = dimensions + i;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 286ff02..53c23b8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -99,7 +99,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
         else {
             ret = cubeDesc.getRowkey().isUseDictionary(col);
             if (ret) {
-                String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+                String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
                 ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
             }
             dictsNeedMerging.put(col, ret);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 68d1481..ea1fbb0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
         else {
             ret = cubeDesc.getRowkey().isUseDictionary(col);
             if (ret) {
-                String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+                String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
                 ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
             }
             dictsNeedMerging.put(col, ret);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index b73fda4..8fced61 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -112,7 +112,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
         CubeDesc cubeDesc = cube.getDescriptor();
 
         for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
-            String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+            String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(),true, col).getTable();
             if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
                 colsNeedMeringDict.add(col);
             } else {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fc016fc..410cec7 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -77,8 +77,7 @@ public class CubeSamplingTest {
     }
 
     private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) {
-        BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-        Integer[] indice = new Integer[bitSet.cardinality()];
+        Integer[] indice = new Integer[Long.bitCount(cuboidId)];
 
         long mask = Long.highestOneBit(baseCuboidId);
         int position = 0;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 01d97fd..32d4d2a 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -229,8 +229,7 @@ public class SparkCubing extends AbstractApplication {
         final ByteArray[] row_hashcodes = new ByteArray[nRowKey];
 
         for (Long cuboidId : allCuboidIds) {
-            BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
-            Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+            Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
 
             long mask = Long.highestOneBit(baseCuboidId);
             int position = 0;
@@ -480,7 +479,7 @@ public class SparkCubing extends AbstractApplication {
         final Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration();
         FileSystem fs = FileSystem.get(hbaseConf);
         FsPermission permission = new FsPermission((short) 0777);
-        for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
+        for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) {
             String cfName = cf.getName();
             Path columnFamilyPath = new Path(hfileLocation, cfName);
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 6b3a82c..9c1d681 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -18,7 +18,6 @@
 package org.apache.kylin.engine.spark.cube;
 
 import java.nio.ByteBuffer;
-import java.util.BitSet;
 import java.util.Map;
 
 import org.apache.kylin.common.util.ByteArray;
@@ -70,7 +69,7 @@ public final class DefaultTupleConverter implements TupleConverter {
         Cuboid cuboid = Cuboid.findById(segment.getCubeDesc(), cuboidId);
         RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
 
-        final int dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+        final int dimensions = Long.bitCount(cuboidId);
         int[] measureColumnsIndex = getMeasureColumnsIndex();
         for (int i = 0; i < measureCount; i++) {
             measureColumnsIndex[i] = dimensions + i;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/examples/test_case_data/localmeta/cube_desc/sample.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/sample.json b/examples/test_case_data/localmeta/cube_desc/sample.json
new file mode 100644
index 0000000..9581b70
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/sample.json
@@ -0,0 +1,257 @@
+{
+  "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+  "name": "test_kylin_cube_with_slr_desc",
+  "description": null,
+  "dimensions": [
+    {
+      "name": "CAL_DT",
+      "table": "EDW.TEST_CAL_DT",
+      "column": null,
+      "derived": [
+        "WEEK_BEG_DT"
+      ],
+    },
+    {
+      "id": 1,
+      "name": "CATEGORY",
+      "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+      "column": null,
+      "derived": [
+        "USER_DEFINED_FIELD1",
+        "USER_DEFINED_FIELD3",
+        "UPD_DATE",
+        "UPD_USER"
+      ],
+      "hierarchy": false
+    },
+    {
+      "id": 2,
+      "name": "CATEGORY_HIERARCHY",
+      "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+      "column": [
+        "META_CATEG_NAME",
+        "CATEG_LVL2_NAME",
+        "CATEG_LVL3_NAME"
+      ],
+      "derived": null,
+      "hierarchy": true
+    },
+    {
+      "id": 3,
+      "name": "LSTG_FORMAT_NAME",
+      "table": "DEFAULT.TEST_KYLIN_FACT",
+      "column": [
+        "LSTG_FORMAT_NAME"
+      ],
+      "derived": null,
+      "hierarchy": false
+    },
+    {
+      "id": 4,
+      "name": "SITE_ID",
+      "table": "EDW.TEST_SITES",
+      "column": null,
+      "derived": [
+        "SITE_NAME",
+        "CRE_USER"
+      ],
+      "hierarchy": false
+    },
+    {
+      "id": 5,
+      "name": "SELLER_TYPE_CD",
+      "table": "EDW.TEST_SELLER_TYPE_DIM",
+      "column": null,
+      "derived": [
+        "SELLER_TYPE_DESC"
+      ],
+      "hierarchy": false
+    },
+    {
+      "id": 6,
+      "name": "SELLER_ID",
+      "table": "DEFAULT.TEST_KYLIN_FACT",
+      "column": [
+        "SELLER_ID"
+      ],
+      "derived": null,
+      "hierarchy": false
+    }
+  ],
+  "measures": [
+    {
+      "id": 1,
+      "name": "GMV_SUM",
+      "function": {
+        "expression": "SUM",
+        "parameter": {
+          "type": "column",
+          "value": "PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      },
+      "dependent_measure_ref": null
+    },
+    {
+      "id": 2,
+      "name": "GMV_MIN",
+      "function": {
+        "expression": "MIN",
+        "parameter": {
+          "type": "column",
+          "value": "PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      },
+      "dependent_measure_ref": null
+    },
+    {
+      "id": 3,
+      "name": "GMV_MAX",
+      "function": {
+        "expression": "MAX",
+        "parameter": {
+          "type": "column",
+          "value": "PRICE"
+        },
+        "returntype": "decimal(19,4)"
+      },
+      "dependent_measure_ref": null
+    },
+    {
+      "id": 4,
+      "name": "TRANS_CNT",
+      "function": {
+        "expression": "COUNT",
+        "parameter": {
+          "type": "constant",
+          "value": "1"
+        },
+        "returntype": "bigint"
+      },
+      "dependent_measure_ref": null
+    },
+    {
+      "id": 5,
+      "name": "ITEM_COUNT_SUM",
+      "function": {
+        "expression": "SUM",
+        "parameter": {
+          "type": "column",
+          "value": "ITEM_COUNT"
+        },
+        "returntype": "bigint"
+      },
+      "dependent_measure_ref": null
+    }
+  ],
+  "rowkey": {
+    "rowkey_columns": [
+      {
+        "column": "seller_id",
+        "length": 18,
+        "dictionary": null,
+        "mandatory": true
+      },
+      {
+        "column": "cal_dt",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "leaf_categ_id",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "meta_categ_name",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "categ_lvl2_name",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "categ_lvl3_name",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "lstg_format_name",
+        "length": 12,
+        "dictionary": null,
+        "mandatory": false
+      },
+      {
+        "column": "lstg_site_id",
+        "length": 0,
+        "dictionary": "true",
+        "mandatory": false
+      },
+      {
+        "column": "slr_segment_cd",
+      }
+    ],
+    "aggregation_groups": [
+      {
+        "include": [
+          "leaf_categ_id",
+          "meta_categ_name",
+          "categ_lvl2_name",
+          "categ_lvl3_name",
+          "cal_dt"
+        ],
+        "selective_rules": {
+          "hierarchy_dims": [
+            [
+              "meta_categ_name",
+              "categ_lvl2_name",
+              "categ_lvl3_name"
+            ]
+          ],
+          "mandatory_dims": [
+            "mandatory"
+          ],
+          "joint_dims": [
+            [
+              "leaf_categ_id",
+              "cal_dt"
+            ]
+          ]
+        }
+      }
+    ]
+  },
+  "last_modified": 1422435345330,
+  "model_name": "test_kylin_inner_join_model_desc",
+  "null_string": null,
+  "hbase_mapping": {
+    "column_family": [
+      {
+        "name": "f1",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "gmv_sum",
+              "gmv_min",
+              "gmv_max",
+              "trans_cnt",
+              "item_count_sum"
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "notify_list": null,
+  "engine_type": 2,
+  "storage_type": 2
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 6670de1..28b0b29 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -194,7 +194,7 @@ public class CubeService extends BasicService {
         }
 
         try {
-            int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc);
+            int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc,false);
             logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
         } catch (Exception e) {
             getCubeDescManager().removeCubeDesc(createdDesc);
@@ -260,7 +260,7 @@ public class CubeService extends BasicService {
             }
 
             CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
-            int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
+            int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc,false);
             logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");
 
             ProjectManager projectManager = getProjectManager();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index c313cc3..be6e22c 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -89,7 +89,7 @@ public class CubeControllerTest extends ServiceTestBase {
         newCube.setModelName(cube.getModelName());
         newCube.setModel(cube.getModel());
         newCube.setDimensions(cube.getDimensions());
-        newCube.setHBaseMapping(cube.getHBaseMapping());
+        newCube.setHbaseMapping(cube.getHbaseMapping());
         newCube.setMeasures(cube.getMeasures());
         newCube.setConfig(cube.getConfig());
         newCube.setRowkey(cube.getRowkey());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index f84e4e6..fcfc335 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -154,7 +154,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         List<HBaseKeyRange> scans = buildScanRanges(flattenToOrAndFilter(filterD), dimensionsD);
 
         // check involved measures, build value decoder for each each family:column
-        List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context);
+        List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHbaseMapping(), metrics, context);
 
         //memory hungry distinct count are pushed down to coprocessor, no need to set threshold any more
         //setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 8813901..6a15bc3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -12,7 +12,6 @@ import org.apache.kylin.common.topn.Counter;
 import org.apache.kylin.common.topn.TopNCounter;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -21,12 +20,9 @@ import org.apache.kylin.cube.kv.RowKeyDecoder;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.metadata.model.DataType;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.StorageContext;
 import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
 import org.apache.kylin.storage.tuple.Tuple;
 import org.apache.kylin.storage.tuple.TupleInfo;
@@ -104,7 +100,7 @@ public class CubeTupleConverter {
         }
         
         // prepare derived columns and filler
-        Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(dimCols, null);
+        Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null);
         for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
             TblColRef[] hostCols = entry.getKey().data;
             for (DeriveInfo deriveInfo : entry.getValue()) {



Mime
View raw message