kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [01/17] kylin git commit: KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler
Date Tue, 22 Aug 2017 09:22:28 GMT
Repository: kylin
Updated Branches:
  refs/heads/2.1.x 69e89b866 -> 61041d746


KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c29283
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c29283
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c29283

Branch: refs/heads/2.1.x
Commit: b5c29283c2e7e6861f6cf9ae87b20825d5ec4a8e
Parents: d0fe948
Author: Li Yang <liyang@apache.org>
Authored: Sun Aug 13 16:34:22 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Sun Aug 13 16:34:22 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/KylinVersion.java   |  24 ++-
 .../apache/kylin/common/KylinVersionTest.java   |  19 +-
 .../cube/cuboid/AggregationGroupScheduler.java  | 126 ------------
 .../kylin/cube/cuboid/CuboidScheduler.java      |   4 +
 .../cube/cuboid/DefaultCuboidScheduler.java     | 191 ++++++++++++++-----
 .../kylin/cube/model/AggregationGroup.java      |  10 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   6 +-
 7 files changed, 187 insertions(+), 193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 960bafe..c6913fc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -37,10 +37,11 @@ public class KylinVersion implements Comparable {
     private static final String COMMIT_SHA1_v15 = "commit_SHA1";
     private static final String COMMIT_SHA1_v13 = "commit.sha1";
 
-    public int major;
-    public int minor;
-    public int revision;
-    public boolean isSnapshot;
+    public final int major;
+    public final int minor;
+    public final int revision;
+    public final int internal;
+    public final boolean isSnapshot;
 
     public KylinVersion(String version) {
 
@@ -58,12 +59,13 @@ public class KylinVersion implements Comparable {
 
         major = Integer.parseInt(splits[0]);
         minor = Integer.parseInt(splits[1]);
-        revision = splits.length < 3 ? 0 : Integer.parseInt(splits[2]);
+        revision = splits.length > 2 ? Integer.parseInt(splits[2]) : 0;
+        internal = splits.length > 3 ? Integer.parseInt(splits[3]) : 0;
     }
 
     @Override
     public String toString() {
-        return "" + major + "." + minor + "." + revision;
+        return "" + major + "." + minor + "." + revision + "." + internal + (isSnapshot ?
"-SNAPSHOT" : "");
     }
 
     @Override
@@ -83,13 +85,17 @@ public class KylinVersion implements Comparable {
         if (comp != 0)
             return comp;
 
+        comp = this.internal - v.internal;
+        if (comp != 0)
+            return comp;
+        
         return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1);
     }
 
     /**
      * Require MANUAL updating kylin version per ANY upgrading.
      */
-    private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0");
+    private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0.20403");
 
     private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
 
@@ -125,6 +131,10 @@ public class KylinVersion implements Comparable {
     public static boolean isBefore200(String ver) {
         return new KylinVersion(ver).compareTo(VERSION_200) < 0;
     }
+    
+    public static int compare(String v1, String v2) {
+        return new KylinVersion(v1).compareTo(new KylinVersion(v2));
+    }
 
     public static void main(String[] args) {
         System.out.println(getKylinClientInformation());

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
index 96b3189..6e903c8 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
@@ -37,4 +37,21 @@ public class KylinVersionTest {
         Assert.assertEquals(1, ver1.minor);
         Assert.assertEquals(0, ver1.revision);
     }
-}
+
+    @Test
+    public void testToString() {
+        KylinVersion ver1 = new KylinVersion("2.1.7.321");
+        Assert.assertEquals(2, ver1.major);
+        Assert.assertEquals(1, ver1.minor);
+        Assert.assertEquals(7, ver1.revision);
+        Assert.assertEquals(321, ver1.internal);
+        Assert.assertEquals("2.1.7.321", ver1.toString());
+    }
+    
+    @Test
+    public void testCompare() {
+        Assert.assertEquals(true, KylinVersion.isBefore200("1.9.9"));
+        Assert.assertEquals(false, KylinVersion.isBefore200("2.0.0"));
+        Assert.assertEquals(true, new KylinVersion("2.1.0").compareTo(new KylinVersion("2.1.0.123"))
< 0);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
deleted file mode 100644
index cdcbcfc..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.cuboid;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import org.apache.kylin.cube.model.AggregationGroup;
-import org.apache.kylin.cube.model.CubeDesc;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class AggregationGroupScheduler {
-    /**
-     * Get all valid cuboids for agg group, ignoring padding
-     * @param agg agg group
-     * @return cuboidId list
-     */
-    public static List<Long> getCuboidsForAgg(final CubeDesc cubeDesc, AggregationGroup
agg) {
-        Set<Long> cuboidHolder = new HashSet<>();
-
-        // build tree structure
-        Set<Long> children = getLowestCuboids(agg);
-        while (!children.isEmpty()) {
-            if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination())
{
-                throw new IllegalStateException("too many combination for the aggregation
group");
-            }
-            cuboidHolder.addAll(children);
-            children = getOnTreeParentsByLayer(children, agg);
-        }
-
-        return Lists.newArrayList(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>()
{
-            @Override
-            public boolean apply(@Nullable Long cuboidId) {
-                return !cubeDesc.isBlackedCuboid(cuboidId);
-            }
-        }));
-    }
-
-    private static Set<Long> getOnTreeParentsByLayer(Collection<Long> children,
final AggregationGroup agg) {
-        Set<Long> parents = new HashSet<>();
-        for (long child : children) {
-            parents.addAll(getOnTreeParents(child, agg));
-        }
-        parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>()
{
-            @Override
-            public boolean apply(@Nullable Long cuboidId) {
-                return agg.checkDimCap(cuboidId);
-            }
-        }));
-        return parents;
-    }
-
-    private static Set<Long> getLowestCuboids(AggregationGroup agg) {
-        return getOnTreeParents(0L, agg);
-    }
-
-    public static Set<Long> getOnTreeParents(long child, AggregationGroup agg) {
-        Set<Long> parentCandidate = new HashSet<>();
-
-        long tmpChild = child;
-        if (tmpChild == agg.getPartialCubeFullMask()) {
-            return parentCandidate;
-        }
-
-        if (agg.getMandatoryColumnMask() != 0L) {
-            if (agg.isMandatoryOnlyValid()) {
-                if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) {
-                    return parentCandidate;
-                }
-            } else {
-                tmpChild |= agg.getMandatoryColumnMask();
-            }
-        }
-
-        for (Long normal : agg.getNormalDims()) {
-            fillBit(tmpChild, normal, parentCandidate);
-        }
-
-        for (Long joint : agg.getJoints()) {
-            fillBit(tmpChild, joint, parentCandidate);
-        }
-
-        for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) {
-            for (long mask : hierarchy.allMasks) {
-                if (fillBit(tmpChild, mask, parentCandidate)) {
-                    break;
-                }
-            }
-        }
-
-        return parentCandidate;
-    }
-
-    private static boolean fillBit(long origin, long other, Set<Long> coll) {
-        if ((origin & other) != other) {
-            coll.add(origin | other);
-            return true;
-        }
-        return false;
-    }
-    
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 1d8f589..e802230 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
 
 import com.google.common.base.Preconditions;
@@ -62,6 +63,9 @@ abstract public class CuboidScheduler {
     
     /** Returns a cuboid on the tree that best matches the request cuboid. */
     abstract public long findBestMatchCuboid(long requestCuboid);
+    
+    /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */
+    abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg);
 
     // ============================================================================
     

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index b75acd5..1c18b69 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -48,7 +48,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     private final Set<Long> allCuboidIds;
     private final Map<Long, List<Long>> parent2child;
     
-
     public DefaultCuboidScheduler(CubeDesc cubeDesc) {
         super(cubeDesc);
         
@@ -84,6 +83,46 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     }
 
     /**
+     * Get the parent cuboid really on the spanning tree.
+     * @param child an on-tree cuboid
+     * @return
+     */
+    @Override
+    public long findBestMatchCuboid(long child) {
+        long parent = getOnTreeParent(child);
+        while (parent > 0) {
+            if (cubeDesc.getAllCuboids().contains(parent)) {
+                break;
+            }
+            parent = getOnTreeParent(parent);
+        }
+
+        if (parent <= 0) {
+            throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+        }
+        return parent;
+    }
+
+    private long getOnTreeParent(long child) {
+        Collection<Long> candidates = getOnTreeParents(child);
+        if (candidates == null || candidates.isEmpty()) {
+            return -1;
+        }
+        return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+    }
+
+    private Set<Long> getOnTreeParents(long child) {
+        List<AggregationGroup> aggrs = Lists.newArrayList();
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            if (agg.isOnTree(child)) {
+                aggrs.add(agg);
+            }
+        }
+
+        return getOnTreeParents(child, aggrs);
+    }
+
+    /**
      * Collect cuboid from bottom up, considering all factor including black list
      * Build tree steps:
      * 1. Build tree from bottom up considering dim capping
@@ -94,7 +133,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
      *    and grandparent are missing, add grandparent to the tree.
      * @return Cuboid collection
      */
-    private Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp()
{
+    protected Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp()
{
         int forward = cubeDesc.getParentForward();
         KylinConfig config = cubeDesc.getConfig();
 
@@ -159,35 +198,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     }
 
     /**
-     * Get the parent cuboid really on the spanning tree.
-     * @param child an on-tree cuboid
-     * @return
-     */
-    @Override
-    public long findBestMatchCuboid(long child) {
-        long parent = getOnTreeParent(child);
-        while (parent > 0) {
-            if (cubeDesc.getAllCuboids().contains(parent)) {
-                break;
-            }
-            parent = getOnTreeParent(parent);
-        }
-
-        if (parent <= 0) {
-            throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
-        }
-        return parent;
-    }
-
-    private long getOnTreeParent(long child) {
-        Collection<Long> candidates = getOnTreeParents(child);
-        if (candidates == null || candidates.isEmpty()) {
-            return -1;
-        }
-        return Collections.min(candidates, Cuboid.cuboidSelectComparator);
-    }
-
-    /**
      * Get all parent for children cuboids, considering dim cap.
      * @param children children cuboids
      * @return all parents cuboids
@@ -205,7 +215,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
                 }
 
                 for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
-                    if (agg.isOnTree(cuboidId) && agg.checkDimCap(cuboidId)) {
+                    if (agg.isOnTree(cuboidId) && checkDimCap(agg, cuboidId)) {
                         return true;
                     }
                 }
@@ -217,22 +227,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     }
 
     /**
-     * Get all *possible* parent for a child cuboid
-     * @param child Child cuboid ID
-     * @return all *possible* parent cuboids
-     */
-    private Set<Long> getOnTreeParents(long child) {
-        List<AggregationGroup> aggrs = Lists.newArrayList();
-        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
-            if (agg.isOnTree(child)) {
-                aggrs.add(agg);
-            }
-        }
-
-        return getOnTreeParents(child, aggrs);
-    }
-
-    /**
      * Get lowest (not Cube building level) Cuboids for every Agg group
      * @return lowest level cuboids
      */
@@ -252,9 +246,108 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
                 parentCandidate.add(Cuboid.getBaseCuboidId(cubeDesc));
                 return parentCandidate;
             }
-            parentCandidate.addAll(AggregationGroupScheduler.getOnTreeParents(child, agg));
+            parentCandidate.addAll(getOnTreeParents(child, agg));
+        }
+
+        return parentCandidate;
+    }
+    
+    /**
+     * Get all valid cuboids for agg group, ignoring padding
+     * @param agg agg group
+     * @return cuboidId list
+     */
+    @Override
+    public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) {
+        Set<Long> cuboidHolder = new HashSet<>();
+
+        // build tree structure
+        Set<Long> children = getLowestCuboids(agg);
+        while (!children.isEmpty()) {
+            if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination())
{
+                throw new IllegalStateException("too many combination for the aggregation
group");
+            }
+            cuboidHolder.addAll(children);
+            children = getOnTreeParentsByLayer(children, agg);
+        }
+
+        return Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>()
{
+            @Override
+            public boolean apply(@Nullable Long cuboidId) {
+                return !cubeDesc.isBlackedCuboid(cuboidId);
+            }
+        }));
+    }
+
+    private Set<Long> getOnTreeParentsByLayer(Collection<Long> children, final
AggregationGroup agg) {
+        Set<Long> parents = new HashSet<>();
+        for (long child : children) {
+            parents.addAll(getOnTreeParents(child, agg));
+        }
+        parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>()
{
+            @Override
+            public boolean apply(@Nullable Long cuboidId) {
+                return checkDimCap(agg, cuboidId);
+            }
+        }));
+        return parents;
+    }
+
+    private Set<Long> getLowestCuboids(AggregationGroup agg) {
+        return getOnTreeParents(0L, agg);
+    }
+
+    private Set<Long> getOnTreeParents(long child, AggregationGroup agg) {
+        Set<Long> parentCandidate = new HashSet<>();
+
+        long tmpChild = child;
+        if (tmpChild == agg.getPartialCubeFullMask()) {
+            return parentCandidate;
+        }
+
+        if (agg.getMandatoryColumnMask() != 0L) {
+            if (agg.isMandatoryOnlyValid()) {
+                if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) {
+                    return parentCandidate;
+                }
+            } else {
+                tmpChild |= agg.getMandatoryColumnMask();
+            }
+        }
+
+        for (Long normal : agg.getNormalDims()) {
+            fillBit(tmpChild, normal, parentCandidate);
+        }
+
+        for (Long joint : agg.getJoints()) {
+            fillBit(tmpChild, joint, parentCandidate);
+        }
+
+        for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) {
+            for (long mask : hierarchy.allMasks) {
+                if (fillBit(tmpChild, mask, parentCandidate)) {
+                    break;
+                }
+            }
         }
 
         return parentCandidate;
     }
+
+    private boolean fillBit(long origin, long other, Set<Long> coll) {
+        if ((origin & other) != other) {
+            coll.add(origin | other);
+            return true;
+        }
+        return false;
+    }
+    
+    private boolean checkDimCap(AggregationGroup agg, long cuboidID) {
+        int dimCap = agg.getDimCap();
+        if (dimCap <= 0)
+            return true;
+        
+        return Long.bitCount(cuboidID) <= dimCap;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 0533ea1..af026af 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -27,7 +27,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.apache.kylin.cube.cuboid.AggregationGroupScheduler;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -296,7 +295,7 @@ public class AggregationGroup implements Serializable {
         long combination = 1;
 
         if (this.getDimCap() > 0) {
-            combination = AggregationGroupScheduler.getCuboidsForAgg(cubeDesc, this).size();
+            combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size();
         } else {
             Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
             Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
@@ -389,13 +388,6 @@ public class AggregationGroup implements Serializable {
         return true;
     }
 
-    public boolean checkDimCap(long cuboidID) {
-        if (getDimCap() <= 0) {
-            return true;
-        }
-        return Long.bitCount(cuboidID) <= getDimCap();
-    }
-
     public void setIncludes(String[] includes) {
         this.includes = includes;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c29283/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
index 80052af..d78da9f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
@@ -18,10 +18,14 @@
 
 package org.apache.kylin.cube.model;
 
+import java.io.Serializable;
+
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-public class SelectRule implements java.io.Serializable {
+public class SelectRule implements Serializable {
+    private static final long serialVersionUID = 1L;
+    
     @JsonProperty("hierarchy_dims")
     public String[][] hierarchyDims;
     @JsonProperty("mandatory_dims")


Mime
View raw message