kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [29/42] kylin git commit: KYLIN-2330 fix CubeDesc returning redundant DeriveInfo
Date Fri, 30 Dec 2016 02:31:21 GMT
KYLIN-2330 fix CubeDesc returning redundant DeriveInfo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a190ecf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a190ecf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a190ecf

Branch: refs/heads/sparkcubing-rebase
Commit: 1a190ecfcaad88e2551c5aff3bd4938719ba46a6
Parents: fdecf09
Author: Yang Li <liyang@apache.org>
Authored: Thu Dec 29 07:13:12 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Thu Dec 29 07:31:17 2016 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/QueryContext.java   |  2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   | 46 +++++++++++++++-----
 .../org/apache/kylin/cube/CubeDescTest.java     | 45 ++++++++++++++++++-
 .../test_case_data/localmeta/cube_desc/ssb.json | 12 ++++-
 4 files changed, 91 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index ef0cb14..93b8556 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -23,7 +23,7 @@ import java.util.Map;
 import com.google.common.collect.Maps;
 
 /**
- * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparison
  */
 public class QueryContext {
     private static final ThreadLocal<Map<String, String>> _queryContext = new
ThreadLocal<Map<String, String>>();

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3b8d034..4686a15 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -807,25 +807,49 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
         for (int i = 0; i < derivedCols.length; i++) {
             if (ArrayUtils.contains(hostCols, derivedCols[i])) {
                 derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
-                extra = (String[]) ArrayUtils.remove(extra, i);
+                if (extra != null)
+                    extra = (String[]) ArrayUtils.remove(extra, i);
                 i--;
             }
         }
+        
+        if (derivedCols.length == 0)
+            return;
 
-        Map<TblColRef, DeriveInfo> toHostMap = derivedToHostMap;
-        Map<Array<TblColRef>, List<DeriveInfo>> hostToMap = hostToDerivedMap;
+        for (int i = 0; i < derivedCols.length; i++) {
+            TblColRef derivedCol = derivedCols[i];
+            boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols,
derivedCol) || (extra != null && extra[i].contains("1-1"));
+            derivedToHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
+        }
 
         Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
-        List<DeriveInfo> infoList = hostToMap.get(hostColArray);
+        List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
         if (infoList == null) {
-            hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
+            hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
         }
-        infoList.add(new DeriveInfo(type, join, derivedCols, false));
-
-        for (int i = 0; i < derivedCols.length; i++) {
-            TblColRef derivedCol = derivedCols[i];
-            boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols,
derivedCol) || (extra != null && extra[i].contains("1-1"));
-            toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
+        
+        // Merged duplicated derived column
+        List<TblColRef> whatsLeft = new ArrayList<>();
+        for (TblColRef derCol : derivedCols) {
+            boolean merged = false;
+            for (DeriveInfo existing : infoList) {
+                if (existing.type == type && existing.join.getPKSide().equals(join.getPKSide()))
{
+                    if (ArrayUtils.contains(existing.columns, derCol)) {
+                        merged = true;
+                        break;
+                    }
+                    if (type == DeriveType.LOOKUP) {
+                        existing.columns = (TblColRef[]) ArrayUtils.add(existing.columns,
derCol);
+                        merged = true;
+                        break;
+                    }
+                }
+            }
+            if (!merged)
+                whatsLeft.add(derCol);
+        }
+        if (whatsLeft.size() > 0) {
+            infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]),
false));
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 0a7adf1..86ea1df 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -23,20 +23,28 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.CubeDesc.DeriveType;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.cube.model.SelectRule;
 import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -331,8 +339,43 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
 
         Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class);
 
-        assertEquals(map, map2);
+        Assert.assertEquals(map, map2);
+    }
 
+    @Test
+    public void testDerivedInfo() {
+        {
+            CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
+            List<TblColRef> givenCols = new ArrayList<>();
+            givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LSTG_SITE_ID"));
+            givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LEAF_CATEG_ID"));
+            Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo =
cube.getHostToDerivedInfo(givenCols, null);
+            assertEquals(3, hostToDerivedInfo.size());
+            assertEquals(Pair.newPair(3, 2), countDerivedInfo(hostToDerivedInfo));
+        }
+
+        {
+            CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ssb");
+            List<TblColRef> givenCols = new ArrayList<>();
+            givenCols.add(cube.findColumnRef("V_LINEORDER", "LO_PARTKEY"));
+            Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo =
cube.getHostToDerivedInfo(givenCols, null);
+            assertEquals(1, hostToDerivedInfo.size());
+            assertEquals(Pair.newPair(1, 1), countDerivedInfo(hostToDerivedInfo));
+        }
+    }
+
+    private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>,
List<DeriveInfo>> hostToDerivedInfo) {
+        int pkfkCount = 0;
+        int lookupCount = 0;
+        for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet())
{
+            for (DeriveInfo deriveInfo : entry.getValue()) {
+                if (deriveInfo.type == DeriveType.PK_FK)
+                    pkfkCount++;
+                if (deriveInfo.type == DeriveType.LOOKUP)
+                    lookupCount++;
+            }
+        }
+        return Pair.newPair(pkfkCount, lookupCount);
     }
 
     private Collection<String> sortStrs(String[] strs) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index d3ea10b..a13ac53 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -6,7 +6,17 @@
     "name" : "SSB.PART_DERIVED",
     "table" : "SSB.PART",
     "column" : null,
-    "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+    "derived" : [ "P_MFGR" ]
+  }, {
+    "name" : "SSB.PART_DERIVED",
+    "table" : "SSB.PART",
+    "column" : null,
+    "derived" : [ "P_CATEGORY" ]
+  }, {
+    "name" : "SSB.PART_DERIVED",
+    "table" : "SSB.PART",
+    "column" : null,
+    "derived" : [ "P_BRAND" ]
   }, {
     "name" : "C_CITY",
     "table" : "SSB.CUSTOMER",


Mime
View raw message