kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1971 bug fix, model matching must match joining tables
Date Sat, 19 Nov 2016 23:49:17 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 634aeaa94 -> 1d4f57af7


KYLIN-1971 bug fix, model matching must match joining tables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d4f57af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d4f57af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d4f57af

Branch: refs/heads/master
Commit: 1d4f57af7cf0ab89a1e1acd10bb4d1aaec1012cf
Parents: 634aeaa
Author: Yang Li <liyang@apache.org>
Authored: Sat Nov 19 21:58:21 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Sun Nov 20 07:48:48 2016 +0800

----------------------------------------------------------------------
 .../kylin/dict/NumberDictionaryBuilder.java     |  3 +
 .../apache/kylin/metadata/model/ColumnDesc.java |  6 ++
 .../kylin/metadata/model/DataModelDesc.java     |  2 +
 .../apache/kylin/metadata/model/JoinDesc.java   | 63 ++++++++++++------
 .../apache/kylin/query/relnode/OLAPJoinRel.java |  1 +
 .../kylin/query/relnode/OLAPTableScan.java      |  1 +
 .../v1/coprocessor/observer/RowTypeTest.java    | 69 --------------------
 7 files changed, 56 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 0655ce1..68a05d4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -21,8 +21,11 @@ package org.apache.kylin.dict;
 import org.apache.kylin.common.util.Bytes;
 
 /**
+ * Use <code>NumberDictionaryForestBuilder</code> instead.
+ * 
  * @author yangli9
  */
+@Deprecated
 public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> {
 
     NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT);

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index e0184b4..7d9133d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -191,6 +191,12 @@ public class ColumnDesc implements Serializable {
         } else if (!name.equals(other.name))
             return false;
 
+        if (table == null) {
+            if (other.table != null)
+                return false;
+        } else if (!table.equals(other.table))
+            return false;
+        
         if (datatype == null) {
             if (other.datatype != null)
                 return false;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7c39a25..139fcbe 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -369,6 +369,8 @@ public class DataModelDesc extends RootPersistentEntity {
                 fkCols[i] = col;
             }
             join.setForeignKeyColumns(fkCols);
+            
+            join.sortByFK();
 
             // Validate join in dimension
             if (pkCols.length != fkCols.length) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 5beda0a..1dd8725 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 
 /**
  */
@@ -98,6 +99,37 @@ public class JoinDesc {
         this.foreignKeyColumns = foreignKeyColumns;
     }
 
+    public void sortByFK() {
+        Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length
== primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length);
+        boolean cont = true;
+        int n = foreignKey.length;
+        for (int i = 0; i < n - 1 && cont; i++) {
+            cont = false;
+            for (int j = i; j < n - 1; j++) {
+                int jj = j + 1;
+                if (foreignKey[j].compareTo(foreignKey[jj]) > 0) {
+                    swap(foreignKey, j, jj);
+                    swap(primaryKey, j, jj);
+                    swap(foreignKeyColumns, j, jj);
+                    swap(primaryKeyColumns, j, jj);
+                    cont = true;
+                }
+            }
+        }
+    }
+
+    private void swap(String[] arr, int j, int jj) {
+        String tmp = arr[j];
+        arr[j] = arr[jj];
+        arr[jj] = tmp;
+    }
+    
+    private void swap(TblColRef[] arr, int j, int jj) {
+        TblColRef tmp = arr[j];
+        arr[j] = arr[jj];
+        arr[jj] = tmp;
+    }
+
     @Override
     public int hashCode() {
         final int prime = 31;
@@ -118,9 +150,10 @@ public class JoinDesc {
             return false;
         JoinDesc other = (JoinDesc) obj;
 
-        if (!this.colRefsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+        // note pk/fk are sorted, sortByFK()
+        if (!Arrays.equals(foreignKeyColumns, other.foreignKeyColumns))
             return false;
-        if (!this.colRefsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+        if (!Arrays.equals(primaryKeyColumns, other.primaryKeyColumns))
             return false;
 
         if (!this.type.equalsIgnoreCase(other.getType()))
@@ -128,39 +161,29 @@ public class JoinDesc {
         return true;
     }
 
-    private boolean colRefsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
-        if (a.length != b.length)
-            return false;
-
-        return Arrays.asList(a).containsAll(Arrays.asList(b));
-    }
-
     // equals() without alias
     public boolean matches(JoinDesc other) {
         if (!this.type.equalsIgnoreCase(other.getType()))
             return false;
-        if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+        
+        // note pk/fk are sorted, sortByFK()
+        if (!this.columnDescEquals(foreignKeyColumns, other.foreignKeyColumns))
             return false;
-        if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+        if (!this.columnDescEquals(primaryKeyColumns, other.primaryKeyColumns))
             return false;
         
         return true;
     }
 
-    private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
+    private boolean columnDescEquals(TblColRef[] a, TblColRef[] b) {
         if (a.length != b.length)
             return false;
         
-        int match = 0;
         for (int i = 0; i < a.length; i++) {
-            for (int j = 0; j < b.length; j++) {
-                if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) {
-                    match++;
-                    break;
-                }
-            }
+            if (a[i].getColumnDesc().equals(b[i].getColumnDesc()) == false)
+                return false;
         }
-        return match == a.length;
+        return true;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 265d65d..4c0a1dc 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -220,6 +220,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
         join.setForeignKeyColumns(fkCols.toArray(new TblColRef[fkCols.size()]));
         join.setPrimaryKey(pks.toArray(COLUMN_ARRAY_MARKER));
         join.setPrimaryKeyColumns(pkCols.toArray(new TblColRef[pkCols.size()]));
+        join.sortByFK();
         return join;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 8b5ad78..e63a978 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -239,6 +239,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel
{
         return new ColumnRowType(columns);
     }
     
+    @SuppressWarnings("deprecation")
     public TblColRef makeRewriteColumn(String name) {
         TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef();
         return tableRef.makeFakeColumn(name);

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
deleted file mode 100644
index 364be9a..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- * 
- */
-public class RowTypeTest extends LocalFileMetadataTestCase {
-
-    @Before
-    public void setUp() throws Exception {
-        this.createTestMetadata();
-    }
-
-    @After
-    public void after() throws Exception {
-        this.cleanupTestMetadata();
-    }
-
-    @Test
-    public void testSerialize() {
-
-        CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
-        CubeDesc cubeDesc = cube.getDescriptor();
-        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid cuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
-        CoprocessorRowType rowType = CoprocessorRowType.fromCuboid(cube.getLatestReadySegment(),
cuboid);
-        byte[] bytes = CoprocessorRowType.serialize(rowType);
-        CoprocessorRowType copy = CoprocessorRowType.deserialize(bytes);
-
-        assertTrue(Arrays.equals(rowType.columnSizes, copy.columnSizes));
-        for (int i = 0; i < rowType.columns.length; i++) {
-            assertEquals(rowType.columns[i].getColumnDesc(), copy.columns[i].getColumnDesc());
-        }
-    }
-}


Mime
View raw message