kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [4/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic
Date Fri, 04 Dec 2015 09:02:36 GMT
http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
new file mode 100644
index 0000000..17f5b2d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ *  find forbid overlaps in each AggregationGroup
+ *  the include dims in AggregationGroup must contain all mandatory, hierarchy and joint
+ */
+public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
+
+    @Override
+    public void validate(CubeDesc cube, ValidateContext context) {
+        inner(cube, context);
+    }
+
+    private int count(String[][] input) {
+        if (input == null) {
+            return 0;
+        } else {
+            int count = 0;
+            for (String[] x : input) {
+                count += count(x);
+            }
+            return count;
+        }
+    }
+
+    private int count(String[] input) {
+        return input == null ? 0 : input.length;
+    }
+
+    private void inner(CubeDesc cube, ValidateContext context) {
+        int maxSize = getMaxAgrGroupSize();
+
+        int index = 0;
+        for (AggregationGroup agg : cube.getAggregationGroups()) {
+            if (agg.getIncludes() == null) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " includes field not set");
+                continue;
+            }
+
+            if (agg.getSelectRule() == null) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " select rule field not set");
+                continue;
+            }
+
+            Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getIncludes() != null) {
+                for (String include : agg.getIncludes()) {
+                    includeDims.add(include);
+                }
+            }
+
+            Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().mandatory_dims != null) {
+                for (String m : agg.getSelectRule().mandatory_dims) {
+                    mandatoryDims.add(m);
+                }
+            }
+
+            Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().hierarchy_dims != null) {
+                for (String[] ss : agg.getSelectRule().hierarchy_dims) {
+                    for (String s : ss)
+                        hierarchyDims.add(s);
+                }
+            }
+
+            Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            if (agg.getSelectRule().joint_dims != null) {
+                for (String[] ss : agg.getSelectRule().joint_dims) {
+                    for (String s : ss)
+                        jointDims.add(s);
+                }
+            }
+
+            if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " Include dims not containing all the used dims");
+                continue;
+            }
+
+            Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+            normalDims.addAll(includeDims);
+            normalDims.removeAll(mandatoryDims);
+            normalDims.removeAll(hierarchyDims);
+            normalDims.removeAll(jointDims);
+
+            int normalDimSize = normalDims.size();
+            int hierarchySize = count(agg.getSelectRule().hierarchy_dims);
+            int jointSize = count(agg.getSelectRule().joint_dims);
+
+            if (normalDimSize + hierarchySize + jointSize > maxSize) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions");
+                continue;
+            }
+
+            if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+                continue;
+            }
+            if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
+                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims");
+                continue;
+            }
+
+            int jointDimNum = 0;
+            if (agg.getSelectRule().joint_dims != null) {
+                for (String[] joints : agg.getSelectRule().joint_dims) {
+
+                    Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+                    for (String s : joints) {
+                        oneJoint.add(s);
+                    }
+
+                    if (oneJoint.size() < 2) {
+                        context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint");
+                        continue;
+                    }
+                    jointDimNum += oneJoint.size();
+
+                    int overlapHierarchies = 0;
+                    if (agg.getSelectRule().hierarchy_dims != null) {
+                        for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) {
+                            Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+                            share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy)));
+
+                            if (!share.isEmpty()) {
+                                overlapHierarchies++;
+                            }
+                            if (share.size() > 1) {
+                                context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy");
+                                continue;
+                            }
+                        }
+
+                        if (overlapHierarchies > 1) {
+                            context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies");
+                            continue;
+                        }
+                    }
+                }
+
+                if (jointDimNum != jointDims.size()) {
+                    context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than one joint");
+                    continue;
+                }
+            }
+
+            index++;
+        }
+    }
+
+    protected int getMaxAgrGroupSize() {
+        String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
+        return Integer.parseInt(size);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index e37b9a9..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- * 
- * @author jianliu
- * 
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
-     * , org.apache.kylin.metadata.validation.ValidateContext)
-     */
-    @Override
-    public void validate(CubeDesc cube, ValidateContext context) {
-        innerValidateMaxSize(cube, context);
-    }
-
-    /**
-     * @param cube
-     * @param context
-     */
-    private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
-        int maxSize = getMaxAgrGroupSize();
-        String[][] groups = cube.getRowkey().getAggregationGroups();
-        for (int i = 0; i < groups.length; i++) {
-            String[] group = groups[i];
-            if (group.length >= maxSize) {
-                context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
-            }
-        }
-    }
-
-    protected int getMaxAgrGroupSize() {
-        String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
-        return Integer.parseInt(size);
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
index 52e5b24..80d4676 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
@@ -20,13 +20,9 @@ package org.apache.kylin.cube.model.validation.rule;
 
 import org.apache.kylin.metadata.MetadataConstants;
 
-/**
- * @author jianliu
- * 
- */
 public interface IKylinValidationConstants extends MetadataConstants {
 
-    public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
+    public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 12;
     public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
     public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index 4585900..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- * 
- * @author jianliu
- * 
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
-     * , org.apache.kylin.metadata.validation.ValidateContext)
-     */
-    @Override
-    public void validate(CubeDesc cube, ValidateContext context) {
-        Set<String> mands = new HashSet<String>();
-        RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
-        if (cols == null || cols.length == 0) {
-            return;
-        }
-        for (int i = 0; i < cols.length; i++) {
-            RowKeyColDesc rowKeyColDesc = cols[i];
-            if (rowKeyColDesc.isMandatory()) {
-                mands.add(rowKeyColDesc.getColumn());
-            }
-        }
-        if (mands.isEmpty()) {
-            return;
-        }
-        String[][] groups = cube.getRowkey().getAggregationGroups();
-        for (int i = 0; i < groups.length; i++) {
-            String[] group = groups[i];
-            for (int j = 0; j < group.length; j++) {
-                String col = group[j];
-                if (mands.contains(col)) {
-                    context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
-                }
-            }
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
index e90f6cd..932754c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.cube.model.validation.rule;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.cube.model.RowKeyDesc;
@@ -28,19 +27,9 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
 
 /**
  * Validate that only one of "length" and "dictionary" appears on rowkey_column
- * 
- * @author jianliu
- * 
  */
 public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
-     * , org.apache.kylin.metadata.validation.ValidateContext)
-     */
     @Override
     public void validate(CubeDesc cube, ValidateContext context) {
         RowKeyDesc row = cube.getRowkey();
@@ -50,25 +39,21 @@ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
         }
 
         RowKeyColDesc[] rcd = row.getRowKeyColumns();
-        if (rcd == null) {
+        if (rcd == null || rcd.length == 0) {
             context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
             return;
         }
-        if (rcd.length == 0) {
-            context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
-            return;
-        }
 
         for (int i = 0; i < rcd.length; i++) {
             RowKeyColDesc rd = rcd[i];
-            if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
-                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
+            if (rd.getColumn() == null || rd.getColumn().length() == 0) {
+                context.addResult(ResultLevel.ERROR, "Rowkey column empty");
+                continue;
             }
-            if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
-                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
+            if (!rd.getEncoding().equalsIgnoreCase("dict") && !rd.getEncoding().toLowerCase().startsWith("fixed_length")) {
+                context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " encoding not dict nor fixed_length");
             }
         }
 
     }
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
new file mode 100644
index 0000000..6366881
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.cube.model.v2.CubeDesc;
+import org.apache.kylin.cube.model.v2.DimensionDesc;
+import org.apache.kylin.cube.model.v2.HBaseMappingDesc;
+import org.apache.kylin.cube.model.v2.RowKeyColDesc;
+import org.apache.kylin.cube.model.v2.RowKeyDesc;
+
+import com.google.common.collect.Lists;
+
+public class CubeDescUpgraderV2 {
+
+    @SuppressWarnings("unused")
+    private static final Log logger = LogFactory.getLog(CubeDescUpgraderV2.class);
+    private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
+
+    private String resourcePath;
+
+    private List<String[]> oldHierarchies = Lists.newArrayList();
+    private List<String> oldMandatories = Lists.newArrayList();
+    private String[][] oldAggGroup = null;
+    private Set<String> allRowKeyCols = newIgnoreCaseSet(null);
+
+    public CubeDescUpgraderV2(String resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
+        CubeDesc oldModel = loadOldCubeDesc(resourcePath);
+
+        org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
+        copyUnChangedProperties(oldModel, newModel);
+        upgradeDimension(oldModel, newModel);
+        upgradeRowKey(oldModel, newModel);
+        upgradeHBaseMapping(oldModel, newModel);
+        upgradeAggregationGroup(newModel);//must do at last
+
+        return newModel;
+    }
+
+    private CubeDesc loadOldCubeDesc(String path) throws IOException {
+        ResourceStore store = getStore();
+
+        CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer);
+
+        if (StringUtils.isBlank(ndesc.getName())) {
+            throw new IllegalStateException("CubeDesc name must not be blank");
+        }
+
+        return ndesc;
+    }
+
+    private Set<String> newIgnoreCaseSet(Set<String> input) {
+        Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+        if (input != null)
+            ret.addAll(input);
+        return ret;
+    }
+
+    private String[] toArray(Set<String> input) {
+        return input.toArray(new String[input.size()]);
+    }
+
+    private boolean rowKeyColExistsInMultipleAggGroup() {
+        if (oldAggGroup == null)
+            return false;
+
+        int total = 0;
+        Set<String> overall = newIgnoreCaseSet(null);
+        for (String[] group : oldAggGroup) {
+            Set<String> temp = newIgnoreCaseSet(null);
+            for (String entry : group) {
+
+                overall.add(entry);
+                temp.add(entry);
+            }
+            total += temp.size();
+        }
+        return overall.size() != total;
+    }
+
+    private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) {
+
+        List<AggregationGroup> aggs = Lists.newArrayList();
+        if (oldAggGroup == null || oldAggGroup.length == 0) {
+            oldAggGroup = new String[1][];
+            oldAggGroup[0] = toArray(allRowKeyCols);
+        }
+
+        if (rowKeyColExistsInMultipleAggGroup()) {
+            throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!");
+        }
+
+        Set<String> visited = newIgnoreCaseSet(null);
+
+        for (String[] group : oldAggGroup) {
+            AggregationGroup agg = new AggregationGroup();
+
+            Set<String> remaining = newIgnoreCaseSet(allRowKeyCols);
+            remaining.removeAll(visited);
+
+            Set<String> joint = newIgnoreCaseSet(remaining);
+            joint.removeAll(oldMandatories);
+
+            Set<String> groupAsSet = newIgnoreCaseSet(null);
+            for (String entry : group) {
+                groupAsSet.add(entry);
+            }
+            visited.addAll(groupAsSet);
+            joint.removeAll(groupAsSet);
+
+            List<String> mandatories = Lists.newArrayList();
+            List<String[]> hierarchies = Lists.newArrayList();
+
+            for (String s : oldMandatories) {
+                mandatories.add(s);
+            }
+
+            for (String[] h : oldHierarchies) {
+                if (groupAsSet.containsAll(Arrays.asList(h))) {
+                    hierarchies.add(h);
+                }
+            }
+
+            agg.setIncludes(toArray(remaining));
+
+            SelectRule selectRule = new SelectRule();
+            selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]);
+            if (joint.size() != 0) {
+                selectRule.joint_dims = new String[1][];
+                selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]);
+            } else {
+                selectRule.joint_dims = new String[0][];
+            }
+            selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]);
+            agg.setSelectRule(selectRule);
+
+            aggs.add(agg);
+
+        }
+        newModel.setAggregationGroups(aggs);
+    }
+
+    private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        List<DimensionDesc> oldDimensions = oldModel.getDimensions();
+        if (oldDimensions == null) {
+            throw new IllegalArgumentException("dimensions is null");
+        }
+        List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList();
+
+        for (DimensionDesc oldDim : oldDimensions) {
+            if (oldDim.isDerived()) {
+                org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+                newDim.setName(oldDim.getName());
+                newDim.setTable(oldDim.getTable());
+                newDim.setColumn("{FK}");
+                newDim.setDerived(oldDim.getDerived());
+
+                newDimensions.add(newDim);
+            } else {
+                if (oldDim.isHierarchy()) {
+                    oldHierarchies.add(oldDim.getColumn());
+                }
+
+                for (String columnStr : oldDim.getColumn()) {
+                    org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+                    newDim.setName(oldDim.getName());
+                    newDim.setTable(oldDim.getTable());
+                    newDim.setColumn(columnStr);
+                    newDim.setDerived(null);
+
+                    newDimensions.add(newDim);
+                }
+            }
+        }
+
+        newModel.setDimensions(newDimensions);
+    }
+
+    private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        RowKeyDesc oldRowKey = oldModel.getRowkey();
+        if (oldModel == null) {
+            throw new IllegalArgumentException("RowKeyDesc is null");
+        }
+
+        if (oldRowKey.getRowKeyColumns() == null) {
+            throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null");
+        }
+
+        org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc();
+        org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length];
+        int index = 0;
+        for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) {
+            org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc();
+
+            allRowKeyCols.add(oldRowKeyCol.getColumn());
+            if (oldRowKeyCol.isMandatory()) {
+                oldMandatories.add(oldRowKeyCol.getColumn());
+            }
+
+            newRowKeyCol.setColumn(oldRowKeyCol.getColumn());
+            if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) {
+                newRowKeyCol.setEncoding("dict");
+            } else if (oldRowKeyCol.getLength() > 0) {
+                newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength());
+            } else {
+                throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength());
+            }
+            cols[index++] = newRowKeyCol;
+        }
+        oldAggGroup = oldRowKey.getAggregationGroups();
+
+        newRowKey.setRowkeyColumns(cols);
+        newModel.setRowkey(newRowKey);
+    }
+
+    private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping();
+        try {
+
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            JsonUtil.writeValueIndent(os, hbaseMappingDesc);
+            byte[] blob = os.toByteArray();
+            ByteArrayInputStream is = new ByteArrayInputStream(blob);
+            org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class);
+            newModel.setHbaseMapping(newHBaseMappingDesc);
+
+        } catch (IOException e) {
+            throw new RuntimeException("error when copying HBaseMappingDesc");
+        }
+    }
+
+    private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+        newModel.setUuid(oldModel.getUuid());
+        newModel.setName(oldModel.getName());
+        newModel.setDescription(oldModel.getDescription());
+        newModel.setMeasures(oldModel.getMeasures());
+        newModel.setNullStrings(oldModel.getNullStrings());
+        newModel.setModelName(oldModel.getModelName());
+        newModel.setNotifyList(oldModel.getNotifyList());
+        newModel.setLastModified(oldModel.getLastModified());
+        newModel.setStorageType(oldModel.getStorageType());
+        newModel.setEngineType(oldModel.getEngineType());
+    }
+
+    protected static ResourceStore getStore() {
+        return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
new file mode 100644
index 0000000..4d3fce9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+
+/**
+ * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1,
+ * that upgrades metadata store from v1(prior kylin 0.7) to v2.
+ * the major difference is that we split cube desc to cube desc + model desc
+ * 
+ * this CubeMetadataUpgradeV2 upgrades metadata store from v2(prior kylin 2.1) to v3
+ * the major different is a brand new definition of partial cubes to allow users to select 
+ * cuboids more flexibly 
+ */
+public class CubeMetadataUpgradeV2 {
+
+    private KylinConfig config = null;
+    private ResourceStore store;
+
+    private List<String> updatedResources = Lists.newArrayList();
+    private List<String> errorMsgs = Lists.newArrayList();
+
+    private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV2.class);
+
+    public CubeMetadataUpgradeV2(String newMetadataUrl) {
+        KylinConfig.destoryInstance();
+        System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl);
+        KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl);
+
+        config = KylinConfig.getInstanceFromEnv();
+        store = getStore();
+    }
+
+    public void upgrade() {
+
+        upgradeCubeDesc();
+        verify();
+    }
+
+    public void verify() {
+        MetadataManager.clearCache();
+        MetadataManager.getInstance(config);
+        CubeDescManager.clearCache();
+        CubeDescManager.getInstance(config);
+        CubeManager.clearCache();
+        CubeManager.getInstance(config);
+        ProjectManager.clearCache();
+        ProjectManager.getInstance(config);
+        //cleanup();
+    }
+
+    private List<String> listResourceStore(String pathRoot) {
+        List<String> paths = null;
+        try {
+            paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX);
+        } catch (IOException e1) {
+            e1.printStackTrace();
+            errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+        }
+
+        return paths;
+    }
+
+    private void upgradeCubeDesc() {
+        logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+
+        List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+        for (String path : paths) {
+
+            try {
+                CubeDescUpgraderV2 upgrade = new CubeDescUpgraderV2(path);
+                CubeDesc ndesc = upgrade.upgrade();
+                ndesc.setSignature(ndesc.calculateSignature());
+
+                getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+                updatedResources.add(ndesc.getResourcePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+                errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
+            }
+        }
+
+    }
+
+    private ResourceStore getStore() {
+        return ResourceStore.getStore(config);
+    }
+
+    public static void main(String[] args) {
+
+        if (!(args != null && (args.length == 1 || args.length == 2))) {
+            System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
+            return;
+        }
+
+        String exportFolder = args[0];
+        boolean verify = false;
+        if (args.length == 2 && "verify".equals(args[1])) {
+            System.out.println("Only verify the metadata in folder " + exportFolder);
+            verify = true;
+        }
+
+        CubeMetadataUpgradeV2 instance = null;
+        if (verify) {
+            instance = new CubeMetadataUpgradeV2(exportFolder);
+            instance.verify();
+        } else {
+            File oldMetaFolder = new File(exportFolder);
+            if (!oldMetaFolder.exists()) {
+                System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
+                return;
+            }
+
+            if (!oldMetaFolder.isDirectory()) {
+                System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
+                return;
+            }
+
+            String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v3";//upgrades metadata store to v3 format
+            try {
+                FileUtils.deleteDirectory(new File(newMetadataUrl));
+                FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+            instance = new CubeMetadataUpgradeV2(newMetadataUrl);
+            instance.upgrade();
+            logger.info("=================================================================");
+            logger.info("Run CubeMetadataUpgrade completed;");
+
+        }
+
+        logger.info("=================================================================");
+        if (instance.errorMsgs.size() > 0) {
+            logger.info("Here are the error/warning messages, you may need check:");
+            for (String s : instance.errorMsgs) {
+                logger.warn(s);
+            }
+        } else {
+            logger.info("No error or warning messages; The migration is success.");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 0cfd020..ada31b6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -34,7 +34,6 @@
 package org.apache.kylin.cube.util;
 
 import java.io.IOException;
-import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -42,14 +41,6 @@ import java.util.Map;
 
 import javax.annotation.Nullable;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
 import org.apache.kylin.common.util.ByteArray;
@@ -59,12 +50,25 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.source.ReadableTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
 /**
  */
 public class CubingUtils {
@@ -82,8 +86,7 @@ public class CubingUtils {
             @Nullable
             @Override
             public Integer[] apply(@Nullable Long cuboidId) {
-                BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-                Integer[] result = new Integer[bitSet.cardinality()];
+                Integer[] result = new Integer[Long.bitCount(cuboidId)];
 
                 long mask = Long.highestOneBit(baseCuboidId);
                 int position = 0;
@@ -100,8 +103,7 @@ public class CubingUtils {
         final Map<Long, HyperLogLogPlusCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
         for (Long cuboidId : allCuboidIds) {
             result.put(cuboidId, new HyperLogLogPlusCounter(14));
-            BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
-            Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+            Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
 
             long mask = Long.highestOneBit(baseCuboidId);
             int position = 0;

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
new file mode 100644
index 0000000..2b75d48
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.junit.Test;
+
+public class AggregationGroupRuleTest {
+
+    @Test
+    public void testGoodDesc() throws IOException {
+        AggregationGroupRule rule = getAggregationGroupRule();
+
+        for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) {
+            CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+            ValidateContext vContext = new ValidateContext();
+            rule.validate(desc, vContext);
+            vContext.print(System.out);
+            assertTrue(vContext.getResults().length == 0);
+        }
+    }
+
+    @Test
+    public void testGoodBecomeBadDesc() throws IOException {
+        AggregationGroupRule rule = new AggregationGroupRule() {
+            @Override
+            protected int getMaxAgrGroupSize() {
+                return 4;
+            }
+        };
+
+        for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) {
+            System.out.println(f.getName());
+            CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+            ValidateContext vContext = new ValidateContext();
+            rule.validate(desc, vContext);
+            vContext.print(System.out);
+            assertEquals(1, vContext.getResults().length);
+            assertEquals("Aggregation group 0 has too many dimensions", (vContext.getResults()[0].getMessage()));
+        }
+    }
+
+    @Test
+    public void testGoodDesc2() throws IOException {
+
+        ValidateContext vContext = new ValidateContext();
+        CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class);
+        desc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {//
+        new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "CATEG_LVL2_NAME" } };
+
+        IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
+        rule.validate(desc, vContext);
+        vContext.print(System.out);
+        assertEquals(0, vContext.getResults().length);
+    }
+
+    @Test
+    public void testBadDesc1() throws IOException {
+
+        ValidateContext vContext = new ValidateContext();
+        CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class);
+        String[] temp = Arrays.asList(desc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]);
+
+        desc.getAggregationGroups().get(0).setIncludes(temp);
+        IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
+        rule.validate(desc, vContext);
+        vContext.print(System.out);
+        //        System.out.println(vContext.getResults().length);
+        //        System.out.println(vContext.getResults()[0].getMessage());
+        assertEquals(1, vContext.getResults().length);
+        assertEquals("Aggregation group 0 Include dims not containing all the used dims", (vContext.getResults()[0].getMessage()));
+    }
+
+    @Test
+    public void testBadDesc2() throws IOException {
+
+        ValidateContext vContext = new ValidateContext();
+        CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class);
+        desc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {//
+        new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "META_CATEG_NAME", "CATEG_LVL2_NAME" } };
+
+        IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
+        rule.validate(desc, vContext);
+        vContext.print(System.out);
+        assertEquals(1, vContext.getResults().length);
+        assertEquals("Aggregation group 0 joint columns overlap with more than 1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
+    }
+
+    public AggregationGroupRule getAggregationGroupRule() {
+        AggregationGroupRule rule = new AggregationGroupRule() {
+            @Override
+            protected int getMaxAgrGroupSize() {
+                return 20;
+            }
+        };
+
+        return rule;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index e7914c7..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- * 
- */
-public class AggregationGroupSizeRuleTest {
-
-    private CubeDesc cube;
-    private ValidateContext vContext = new ValidateContext();
-
-    /**
-     * @throws java.lang.Exception
-     */
-    @Before
-    public void setUp() throws Exception {
-        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
-        this.cube = desc2;
-
-    }
-
-    @Test
-    public void testOneMandatoryColumn() {
-        IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
-            /*
-             * (non-Javadoc)
-             * 
-             * @see
-             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
-             * #getMaxAgrGroupSize()
-             */
-            @Override
-            protected int getMaxAgrGroupSize() {
-                return 3;
-            }
-        };
-        rule.validate(cube, vContext);
-        vContext.print(System.out);
-        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
-        assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
-        assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[2].getMessage()
-        // .startsWith("Hierachy column"));
-    }
-
-    @Test
-    public void testAggColumnSize() {
-        AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
-            /*
-             * (non-Javadoc)
-             * 
-             * @see
-             * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
-             * #getMaxAgrGroupSize()
-             */
-            @Override
-            protected int getMaxAgrGroupSize() {
-                return 20;
-            }
-        };
-        rule.validate(cube, vContext);
-        vContext.print(System.out);
-        assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[0].getMessage()
-        // .startsWith("Aggregation group"));
-        // assertTrue("Failed to validate aggragation group error",
-        // vContext.getResults()[0].getMessage()
-        // .startsWith("Hierachy column"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index fcfa67d..15c6deb 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -66,7 +66,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
 
         String signature = desc.calculateSignature();
         desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column");
-        assertTrue(!signature.equals(desc.calculateSignature()));
+        assertTrue(signature.equals(desc.calculateSignature()));
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index d7feb56..0de5c22 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -56,10 +56,10 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
         CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
         TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
 
-        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
         System.out.println(JsonUtil.writeValueAsIndentString(info1));
 
-        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
         System.out.println(JsonUtil.writeValueAsIndentString(info2));
 
         assertTrue(info1.getUuid() == info2.getUuid());

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
deleted file mode 100644
index 29b7183..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- * 
- */
-public class MandatoryColumnRuleTest {
-
-    private CubeDesc cube;
-    private ValidateContext vContext = new ValidateContext();
-
-    /**
-     * @throws java.lang.Exception
-     */
-    @Before
-    public void setUp() throws Exception {
-        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
-        this.cube = desc2;
-
-    }
-
-    @Test
-    public void testOneMandatoryColumn() {
-        IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
-        rule.validate(cube, vContext);
-        assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
-        assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
index 000fa1d..be7b624 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
@@ -20,39 +20,42 @@ package org.apache.kylin.cube;
 
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
-import org.junit.Before;
 import org.junit.Test;
 
-/**
- * @author jianliu
- * 
- */
 public class RowKeyAttrRuleTest {
 
-    private CubeDesc cube;
-    private ValidateContext vContext = new ValidateContext();
-
-    /**
-     * @throws java.lang.Exception
-     */
-    @Before
-    public void setUp() throws Exception {
-        CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class);
-        this.cube = desc2;
-
+    @Test
+    public void testGoodDesc() throws IOException {
+        for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) {
+            CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+            ValidateContext vContext = new ValidateContext();
+            IValidatorRule<CubeDesc> rule = new RowKeyAttrRule();
+            rule.validate(desc, vContext);
+            vContext.print(System.out);
+            assertTrue(vContext.getResults().length == 0);
+        }
     }
 
     @Test
-    public void testOneMandatoryColumn() {
+    public void testBadDesc() throws IOException {
+        ValidateContext vContext = new ValidateContext();
+        CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class);
+        desc.getRowkey().getRowKeyColumns()[1].setEncoding("non-supported-encoding");
+        desc.getRowkey().getRowKeyColumns()[2].setColumn("");
         IValidatorRule<CubeDesc> rule = new RowKeyAttrRule();
-        rule.validate(cube, vContext);
+        rule.validate(desc, vContext);
         vContext.print(System.out);
-        assertTrue("Failed to validate rowkey", vContext.getResults().length == 1);
-        assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column"));
+        assertTrue(vContext.getResults().length == 2);
+        assertTrue("Rowkey column cal_dt encoding not dict nor fixed_length".equals(vContext.getResults()[0].getMessage()));
+        assertTrue("Rowkey column empty".equalsIgnoreCase(vContext.getResults()[1].getMessage()));
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 83e8d94..0711d6e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -18,14 +18,12 @@
 
 package org.apache.kylin.cube.cuboid;
 
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -79,37 +77,16 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_desc");
     }
 
+    private CubeDesc getTestKylinCubeWithSellerLeft() {
+        return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_left_join_desc");
+    }
+
     private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() {
         return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
     }
 
-    @Test
-    public void testFindSmallerSibling1() {
-        CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler scheduler = new CuboidScheduler(cube);
-
-        Collection<Long> siblings;
-
-        siblings = scheduler.findSmallerSibling(255);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(133);
-        assertEquals("[131]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(127);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(134);
-        assertEquals("[131, 133]", sortToString(siblings));
-
-        siblings = scheduler.findSmallerSibling(130);
-        assertEquals("[129]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(5);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(135);
-        assertEquals("[]", siblings.toString());
+    private CubeDesc getStreamingCubeDesc() {
+        return getCubeDescManager().getCubeDesc("test_streaming_table_cube_desc");
     }
 
     private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
@@ -124,32 +101,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     }
 
     @Test
-    public void testFindSmallerSibling2() {
-        CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler scheduler = new CuboidScheduler(cube);
-
-        Collection<Long> siblings;
-
-        siblings = scheduler.findSmallerSibling(511);
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("110111111"));
-        assertEquals("[383]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("101110111"));
-        assertEquals("[319]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("111111000"));
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("111111000"));
-        assertEquals("[]", siblings.toString());
-
-        siblings = scheduler.findSmallerSibling(toLong("110000000"));
-        assertEquals("[288, 320]", sortToString(siblings));
-    }
-
-    @Test
     public void testGetSpanningCuboid2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
         CuboidScheduler scheduler = new CuboidScheduler(cube);
@@ -183,7 +134,12 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         CuboidScheduler scheduler = new CuboidScheduler(cube);
 
         long quiz = toLong("01100111");
-        testSpanningAndGetParent(scheduler, cube, new long[] { quiz });
+        try {
+            testSpanningAndGetParent(scheduler, cube, new long[] { quiz });
+            fail();
+        } catch (IllegalStateException ex) {
+            //expected
+        }
 
         // generate 7d
         System.out.println("Spanning for 7D Cuboids");
@@ -210,16 +166,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     }
 
     @Test
-    public void testGetSpanningCuboid() {
-        CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler scheduler = new CuboidScheduler(cube);
-
-        Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(248);
-
-        assertEquals("[]", spnanningCuboids.toString());
-    }
-
-    @Test
     public void testGetCardinality() {
         CubeDesc cube = getTestKylinCubeWithSeller();
         CuboidScheduler scheduler = new CuboidScheduler(cube);
@@ -233,43 +179,98 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
 
     @Test
     public void testCuboidGeneration1() {
+
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
     public void testCuboidGeneration2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
     public void testCuboidGeneration3() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
-        CuboidCLI.simulateCuboidGeneration(cube);
+        CuboidCLI.simulateCuboidGeneration(cube, true);
+    }
+
+    @Test
+    public void testCuboidGeneration4() {
+        CubeDesc cube = getTestKylinCubeWithSellerLeft();
+        CuboidCLI.simulateCuboidGeneration(cube, true);
+    }
+
+    @Test
+    public void testCuboidGeneration5() {
+        CubeDesc cube = getStreamingCubeDesc();
+        CuboidCLI.simulateCuboidGeneration(cube, true);
     }
 
     @Test
     public void testCuboidCounts1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
-        assertArrayEquals(new int[] { 1, 4, 6, 6, 4, 4, 2, 0 }, counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
     }
 
     @Test
     public void testCuboidCounts2() {
+        CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+        int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+        printCount(counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
+    }
+
+    @Test
+    public void testCuboidCounts3() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidCLI.calculateAllLevelCount(cube);
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+        int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+        printCount(counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
+    }
+
+    @Test
+    public void testCuboidCounts4() {
+        CubeDesc cube = getTestKylinCubeWithSellerLeft();
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
-        assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
     }
 
-    private String sortToString(Collection<Long> longs) {
-        ArrayList<Long> copy = new ArrayList<Long>(longs);
-        Collections.sort(copy);
-        return copy.toString();
+    @Test
+    public void testCuboidCounts5() {
+        CubeDesc cube = getStreamingCubeDesc();
+        CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+        int[] counts = CuboidCLI.calculateAllLevelCount(cube);
+        printCount(counts);
+        int sum = 0;
+        for (Integer x : counts) {
+            sum += x;
+        }
+        assertEquals(cuboidScheduler.getCuboidCount(), sum);
     }
 
     public CubeDescManager getCubeDescManager() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 89e235f..eecf368 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.cube.cuboid;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -116,12 +115,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testIsValid2() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        try {
-            assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
-            fail();
-        } catch (IllegalArgumentException ex) {
-            // expected
-        }
+        assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
 
         // base
         assertEquals(false, Cuboid.isValid(cube, 0));
@@ -148,7 +142,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, 0);
-        assertEquals(toLong("100100000"), cuboid.getId());
+        assertEquals(toLong("101000000"), cuboid.getId());
 
         cuboid = Cuboid.findById(cube, 1);
         assertEquals(toLong("100000111"), cuboid.getId());

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/resources/data/TEST1_desc.json
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/data/TEST1_desc.json b/core-cube/src/test/resources/data/TEST1_desc.json
deleted file mode 100644
index 082a3cf..0000000
--- a/core-cube/src/test/resources/data/TEST1_desc.json
+++ /dev/null
@@ -1,183 +0,0 @@
-{
-  "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
-  "last_modified" : 1401429176099,
-  "name" : "TEST1_desc",
-  "model_name": "TEST1_model_desc",
-  "fact_table" : "TEST_KYLIN_FACT",
-  "dimensions" : [ {
-    "id" : 1,
-    "name" : "CAL_DT",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "CAL_DT" ],
-      "foreign_key" : [ "CAL_DT" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_CAL_DT",
-    "column" : ["TEST_CAL_DT.CAL_DT"],
-    "derived" : [ "WEEK_BEG_DT" ]
-  }, {
-    "id" : 2,
-    "name" : "CATEGORY",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
-      "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : true,
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"],
-    "datatype" : null,
-    "derived" : null
-  }, {
-    "id" : 3,
-    "name" : "LSTG_FORMAT_NAME",
-    "join" : null,
-    "hierarchy" : null,
-    "table" : "TEST_KYLIN_FACT",
-    "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"],
-    "datatype" : "string",
-    "derived" : null
-  }, {
-    "id" : 4,
-    "name" : "SITE_ID",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SITE_ID" ],
-      "foreign_key" : [ "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_SITES",
-    "column" : ["TEST_SITES.SITE_ID"],
-    "datatype" : "string",
-    "derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-    "id" : 5,
-    "name" : "SELLER_TYPE_CD",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SELLER_TYPE_CD" ],
-      "foreign_key" : [ "SLR_SEGMENT_CD" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_SELLER_TYPE_DIM",
-    "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"],
-    "datatype" : "string",
-    "derived" : [ "SELLER_TYPE_DESC" ]
-  } ],
-  "measures" : [ {
-    "id" : 1,
-    "name" : "GMV_SUM",
-    "function" : {
-      "expression" : "SUM",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 2,
-    "name" : "GMV_MIN",
-    "function" : {
-      "expression" : "MIN",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 3,
-    "name" : "GMV_MAX",
-    "function" : {
-      "expression" : "MAX",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 4,
-    "name" : "TRANS_CNT",
-    "function" : {
-      "expression" : "COUNT",
-      "parameter" : {
-        "type" : "constant",
-        "value" : "1"
-      },
-      "returntype" : "long"
-    }
-  }, {
-    "id" : 5,
-    "name" : "SELLER_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  }, {
-    "id" : 6,
-    "name" : "SELLER_FORMAT_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "LSTG_FORMAT_NAME,SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  } ],
-  "rowkey" : {
-    "rowkey_columns" : [ {
-      "column" : "CAL_DT",
-      "length" : 0,
-      "dictionary" : "date(yyyy-mm-dd)",
-      "mandatory" : true
-    }, {
-      "column" : "META_CATEG_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL2_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL3_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "LSTG_FORMAT_NAME",
-      "length" : 12,
-      "dictionary" : null,
-      "mandatory" : false
-    }, {
-      "column" : "SITE_ID",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "SELLER_TYPE_CD",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    } ],
-    "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ]
-  },
-  "hbase_mapping" : {
-    "column_family" : [ {
-      "name" : "F1",
-      "columns" : [ {
-        "qualifier" : "M",
-        "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ]
-      } ]
-    } ]
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/resources/data/TEST2_desc.json
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/data/TEST2_desc.json b/core-cube/src/test/resources/data/TEST2_desc.json
deleted file mode 100644
index 90bd806..0000000
--- a/core-cube/src/test/resources/data/TEST2_desc.json
+++ /dev/null
@@ -1,168 +0,0 @@
-{
-  "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
-  "last_modified" : 1401429176099,
-  "model_name": "TEST2_model_desc",
-  "name" : "TEST2_desc",
-  "dimensions" : [ {
-    "name" : "CAL_DT",
-    "hierarchy" : null,
-    "column" : ["TEST_CAL_DT.CAL_DT"],
-    "derived" : [ "WEEK_BEG_DT" ]
-  }, {
-    "id" : 2,
-    "name" : "CATEGORY",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
-      "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : true,
-    "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"],
-    "derived" : null
-  }, {
-    "id" : 3,
-    "name" : "LSTG_FORMAT_NAME",
-    "hierarchy" : null,
-    "table" : "TEST_KYLIN_FACT",
-    "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"],
-    "derived" : null
-  }, {
-    "id" : 4,
-    "name" : "SITE_ID",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SITE_ID" ],
-      "foreign_key" : [ "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : null,
-    "column" : ["TEST_SITES.SITE_ID"],
-    "derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-    "id" : 5,
-    "name" : "SELLER_TYPE_CD",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SELLER_TYPE_CD" ],
-      "foreign_key" : [ "SLR_SEGMENT_CD" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_SELLER_TYPE_DIM",
-    "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"],
-    "derived" : [ "SELLER_TYPE_DESC" ]
-  } ],
-  "measures" : [ {
-    "id" : 1,
-    "name" : "GMV_SUM",
-    "function" : {
-      "expression" : "SUM",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 2,
-    "name" : "GMV_MIN",
-    "function" : {
-      "expression" : "MIN",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 3,
-    "name" : "GMV_MAX",
-    "function" : {
-      "expression" : "MAX",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 4,
-    "name" : "TRANS_CNT",
-    "function" : {
-      "expression" : "COUNT",
-      "parameter" : {
-        "type" : "constant",
-        "value" : "1"
-      },
-      "returntype" : "long"
-    }
-  }, {
-    "id" : 5,
-    "name" : "SELLER_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  }, {
-    "id" : 6,
-    "name" : "SELLER_FORMAT_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "LSTG_FORMAT_NAME,SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  } ],
-  "rowkey" : {
-    "rowkey_columns" : [ {
-      "column" : "CAL_DT",
-      "length" : 0,
-      "dictionary" : "date(yyyy-mm-dd)",
-      "mandatory" : true
-    }, {
-      "column" : "META_CATEG_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL2_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL3_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "LSTG_FORMAT_NAME",
-      "length" : 12,
-      "dictionary" : null,
-      "mandatory" : false
-    }, {
-      "column" : "SITE_ID",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "SELLER_TYPE_CD",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    } ],
-    "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD", "CATEG_LVL2_NAME" ] ]
-  },
-  "hbase_mapping" : {
-    "column_family" : [ {
-      "name" : "F1",
-      "columns" : [ {
-        "qualifier" : "M",
-        "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ]
-      } ]
-    } ]
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/test/resources/data/TEST3_desc.json
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/data/TEST3_desc.json b/core-cube/src/test/resources/data/TEST3_desc.json
deleted file mode 100644
index 4b0836b..0000000
--- a/core-cube/src/test/resources/data/TEST3_desc.json
+++ /dev/null
@@ -1,182 +0,0 @@
-{
-  "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
-  "last_modified" : 1401429176099,
-  "name" : "TEST1_desc",
-  "model_name": "TEST1_model_desc",
-  "dimensions" : [ {
-    "id" : 1,
-    "name" : "CAL_DT",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "CAL_DT" ],
-      "foreign_key" : [ "CAL_DT" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_CAL_DT",
-    "column" : ["TEST_CAL_DT.CAL_DT"],
-    "datatype" : "date",
-    "derived" : [ "WEEK_BEG_DT" ]
-  }, {
-    "id" : 2,
-    "name" : "CATEGORY",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
-      "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : true,
-    "table" : "TEST_CATEGORY_GROUPINGS",
-    "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"],
-    "derived" : null
-  }, {
-    "id" : 3,
-    "name" : "LSTG_FORMAT_NAME",
-    "join" : null,
-    "hierarchy" : null,
-    "table" : "TEST_KYLIN_FACT",
-    "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"],
-    "datatype" : "string",
-    "derived" : null
-  }, {
-    "id" : 4,
-    "name" : "SITE_ID",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SITE_ID" ],
-      "foreign_key" : [ "LSTG_SITE_ID" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_SITES",
-    "column" : ["TEST_SITES.SITE_ID"],
-    "datatype" : "string",
-    "derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-    "id" : 5,
-    "name" : "SELLER_TYPE_CD",
-    "join" : {
-      "type" : "inner",
-      "primary_key" : [ "SELLER_TYPE_CD" ],
-      "foreign_key" : [ "SLR_SEGMENT_CD" ]
-    },
-    "hierarchy" : null,
-    "table" : "TEST_SELLER_TYPE_DIM",
-    "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"],
-    "datatype" : "string",
-    "derived" : [ "SELLER_TYPE_DESC" ]
-  } ],
-  "measures" : [ {
-    "id" : 1,
-    "name" : "GMV_SUM",
-    "function" : {
-      "expression" : "SUM",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 2,
-    "name" : "GMV_MIN",
-    "function" : {
-      "expression" : "MIN",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 3,
-    "name" : "GMV_MAX",
-    "function" : {
-      "expression" : "MAX",
-      "parameter" : {
-        "type" : "column",
-        "value" : "PRICE"
-      },
-      "returntype" : "decimal"
-    }
-  }, {
-    "id" : 4,
-    "name" : "TRANS_CNT",
-    "function" : {
-      "expression" : "COUNT",
-      "parameter" : {
-        "type" : "constant",
-        "value" : "1"
-      },
-      "returntype" : "long"
-    }
-  }, {
-    "id" : 5,
-    "name" : "SELLER_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  }, {
-    "id" : 6,
-    "name" : "SELLER_FORMAT_CNT",
-    "function" : {
-      "expression" : "COUNT_DISTINCT",
-      "parameter" : {
-        "type" : "column",
-        "value" : "LSTG_FORMAT_NAME,SELLER_ID"
-      },
-      "returntype" : "hllc10"
-    }
-  } ],
-  "rowkey" : {
-    "rowkey_columns" : [ {
-      "column" : "CAL_DT",
-      "length" : 10,
-      "dictionary" : "date(yyyy-mm-dd)",
-      "mandatory" : false
-    }, {
-      "column" : "META_CATEG_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL2_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "CATEG_LVL3_NAME",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "LSTG_FORMAT_NAME",
-      "length" : 12,
-      "dictionary" : null,
-      "mandatory" : false
-    }, {
-      "column" : "SITE_ID",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    }, {
-      "column" : "SELLER_TYPE_CD",
-      "length" : 0,
-      "dictionary" : "string",
-      "mandatory" : false
-    } ],
-    "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ]
-  },
-  "hbase_mapping" : {
-    "column_family" : [ {
-      "name" : "F1",
-      "columns" : [ {
-        "qualifier" : "M",
-        "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ]
-      } ]
-    } ]
-  }
-}
\ No newline at end of file


Mime
View raw message