kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1875 fix PK-FK derive from each other
Date Thu, 01 Dec 2016 13:43:05 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 a9100bfc6 -> 48253d1ff


KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48253d1f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48253d1f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48253d1f

Branch: refs/heads/KYLIN-1875
Commit: 48253d1ff03e137a8568b8ca880c45b5b8c524f3
Parents: a9100bf
Author: Yang Li <liyang@apache.org>
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Thu Dec 1 21:42:52 2016 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++++++++++---------
 .../storage/gtrecord/CubeTupleConverter.java    |  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java        |  2 +-
 .../translate/DerivedFilterTranslator.java      |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java    |  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java       |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..6a25594 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -657,10 +657,10 @@ public class CubeManager implements IRealizationProvider {
         cubeMap.removeLocal(cubeName);
     }
 
-    public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
+    public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
 
-        String tableName = dim.getTableRef().getTableIdentity();
-        String[] pkCols = dim.getJoin().getPrimaryKey();
+        String tableName = join.getPKSide().getTableIdentity();
+        String[] pkCols = join.getPrimaryKey();
         String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
         if (snapshotResPath == null)
             throw new IllegalStateException("No snaphot for table '" + tableName + "' found
on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index fa4da92..18b7e7a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
@@ -98,20 +99,20 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
 
     public static class DeriveInfo {
         public DeriveType type;
-        public DimensionDesc dimension;
+        public JoinDesc join;
         public TblColRef[] columns;
         public boolean isOneToOne; // only used when ref from derived to host
 
-        DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean
isOneToOne) {
+        DeriveInfo(DeriveType type, JoinDesc join, TblColRef[] columns, boolean isOneToOne)
{
             this.type = type;
-            this.dimension = dimension;
+            this.join = join;
             this.columns = columns;
             this.isOneToOne = isOneToOne;
         }
 
         @Override
         public String toString() {
-            return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns="
+ Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
+            return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns)
+ ", isOneToOne=" + isOneToOne + "]";
         }
 
     }
@@ -729,27 +730,34 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
                 for (int i = 0; i < derivedNames.length; i++) {
                     derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
                 }
-                initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+                initDerivedMap(dimColArray, DeriveType.LOOKUP, join, derivedCols, derivedExtra);
             }
-
-            // PK-FK derive the other side
+            
             if (join != null) {
-                TblColRef[] fk = join.getForeignKeyColumns();
-                TblColRef[] pk = join.getPrimaryKeyColumns();
-
-                allColumns.addAll(Arrays.asList(fk));
-                allColumns.addAll(Arrays.asList(pk));
-                for (int i = 0; i < fk.length; i++) {
-                    int find = ArrayUtils.indexOf(dimColArray, fk[i]);
-                    if (find >= 0) {
-                        TblColRef derivedCol = initDimensionColRef(pk[i]);
-                        initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK,
dim, new TblColRef[] { derivedCol }, null);
-                    }
+                allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
+                allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
+            }
+        }
+        
+        // PK-FK derive the other side
+        Set<TblColRef> realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true));
+        for (JoinTableDesc joinTable : model.getJoinTables()) {
+            JoinDesc join = joinTable.getJoin();
+            int n = join.getForeignKeyColumns().length;
+            for (int i = 0; i < n; i++) {
+                TblColRef pk = join.getPrimaryKeyColumns()[i];
+                TblColRef fk = join.getForeignKeyColumns()[i];
+                if (realDimensions.contains(pk) && !realDimensions.contains(fk))
{
+                    initDimensionColRef(fk);
+                    initDerivedMap(new TblColRef[] { pk }, DeriveType.PK_FK, join, new TblColRef[]
{ fk }, null);
+                } else if (realDimensions.contains(fk) && !realDimensions.contains(pk))
{
+                    initDimensionColRef(pk);
+                    initDerivedMap(new TblColRef[] { fk }, DeriveType.PK_FK, join, new TblColRef[]
{ pk }, null);
                 }
             }
         }
     }
-
+    
     private String[][] splitDerivedColumnAndExtra(String[] derived) {
         String[] cols = new String[derived.length];
         String[] extra = new String[derived.length];
@@ -767,7 +775,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
         return new String[][] { cols, extra };
     }
 
-    private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension,
TblColRef[] derivedCols, String[] extra) {
+    private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[]
derivedCols, String[] extra) {
         if (hostCols.length == 0 || derivedCols.length == 0)
             throw new IllegalStateException("host/derived columns must not be empty");
 
@@ -790,12 +798,12 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
         if (infoList == null) {
             hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
         }
-        infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
+        infoList.add(new DeriveInfo(type, join, derivedCols, false));
 
         for (int i = 0; i < derivedCols.length; i++) {
             TblColRef derivedCol = derivedCols[i];
             boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols,
derivedCol) || (extra != null && extra[i].contains("1-1"));
-            toHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
+            toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 7ec24b2..3159318 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -266,7 +266,7 @@ public class CubeTupleConverter {
             case LOOKUP:
                 return new IDerivedColumnFiller() {
                     CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
-                    LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+                    LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
                     int[] derivedColIdx = initDerivedColIdx();
                     Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 9c74cca..31573d0 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -294,7 +294,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
         CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
         CubeSegment seg = cubeInstance.getLatestReadySegment();
-        LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+        LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
         Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup,
hostInfo, compf);
         TupleFilter translatedFilter = translated.getFirst();
         boolean loosened = translated.getSecond();

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 160338f..13c655c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -54,7 +54,7 @@ public class DerivedFilterTranslator {
 
         TblColRef derivedCol = compf.getColumn();
         TblColRef[] hostCols = hostInfo.columns;
-        TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns();
+        TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns();
 
         if (hostInfo.type == DeriveType.PK_FK) {
             assert hostCols.length == 1;

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 90324b5..6d91753 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
     @Rule

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 1c2bc2d..28ee623 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -67,7 +67,7 @@ public class LookupTableEnumerator implements Enumerator<Object[]>
{
             throw new IllegalStateException("No dimension with derived columns found for
lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());
 
         CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
-        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
+        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim.getJoin());
         this.allRows = table.getAllRows();
 
         OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 9af0faf..ca46188 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -368,7 +368,7 @@ public class CubeStorageQuery implements IStorageQuery {
         DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
         CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
         CubeSegment seg = cubeInstance.getLatestReadySegment();
-        LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+        LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
         Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup,
hostInfo, compf);
         TupleFilter translatedFilter = translated.getFirst();
         boolean loosened = translated.getSecond();

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index d2378b9..64feff0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -220,7 +220,7 @@ public class CubeTupleConverter {
         case LOOKUP:
             return new IDerivedColumnFiller() {
                 CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
-                LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+                LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
                 int[] derivedColIdx = initDerivedColIdx();
                 Array<String> lookupKey = new Array<String>(new String[hostColIdx.length]);
 


Mime
View raw message