kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [21/50] [abbrv] kylin git commit: KYLIN-2012 adjust rules for LookupTable & remove dead methods
Date Wed, 19 Oct 2016 03:02:08 GMT
KYLIN-2012 adjust rules for LookupTable & remove dead methods


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36cf99ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36cf99ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36cf99ef

Branch: refs/heads/master-hbase1.x
Commit: 36cf99ef77486c1361a31f3e1f748bb277eca217
Parents: 96935fe
Author: gaodayue <gaodayue@meituan.com>
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: gaodayue <gaodayue@meituan.com>
Committed: Thu Oct 13 16:56:35 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/metadata/MetadataManager.java  | 30 --------------------
 .../apache/kylin/source/hive/SchemaChecker.java | 27 ++++++++++++++----
 2 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 4e9c1ed..92d41b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
         reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
     }
 
-    public void reloadTableCache(String tableIdentity) throws IOException {
-        reloadSourceTableExt(tableIdentity);
-        reloadSourceTable(tableIdentity);
-    }
-
     public DataModelDesc getDataModelDesc(String name) {
         return dataModelDescMap.get(name);
     }
@@ -622,29 +617,4 @@ public class MetadataManager {
         srcTableExdMap.remove(tableIdentity);
     }
 
-    public String appendDBName(String table) {
-
-        if (table.indexOf(".") > 0)
-            return table;
-
-        Map<String, TableDesc> map = getAllTablesMap();
-
-        int count = 0;
-        String result = null;
-        for (TableDesc t : map.values()) {
-            if (t.getName().equalsIgnoreCase(table)) {
-                result = t.getIdentity();
-                count++;
-            }
-        }
-
-        if (count == 1)
-            return result;
-
-        if (count > 1) {
-            logger.warn("There are more than 1 table named with '" + table + "' in different
database; The program couldn't determine, randomly pick '" + result + "'");
-        }
-        return result;
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e955410..319ebee 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -152,7 +152,14 @@ public class SchemaChecker {
         }
     }
 
-    private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String,
FieldSchema> fieldsMap) {
+    /**
+     * check whether all columns used in `cube` has compatible schema in current hive schema
denoted by `fieldsMap`.
+     * @param cube cube to check, must use `table` in its model
+     * @param table kylin's table metadata
+     * @param fieldsMap current hive schema of `table`
+     * @return true if all columns used in `cube` has compatible schema with `fieldsMap`,
false otherwise
+     */
+    private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table,
Map<String, FieldSchema> fieldsMap) {
         Set<ColumnDesc> usedColumns = Sets.newHashSet();
         for (TblColRef col : cube.getAllColumns()) {
             usedColumns.add(col.getColumnDesc());
@@ -170,8 +177,15 @@ public class SchemaChecker {
         return violateColumns;
     }
 
-    private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) {
-        if (table.getColumnCount() != fields.size()) {
+    /**
+     * check whether all columns in `table` are still in `fields` and have the same index
as before.
+     *
+     * @param table kylin's table metadata
+     * @param fields current table metadata in hive
+     * @return true if only new columns are appended in hive, false otherwise
+     */
+    private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields)
{
+        if (table.getColumnCount() > fields.size()) {
             return false;
         }
 
@@ -212,15 +226,16 @@ public class SchemaChecker {
             // if user reloads a fact table used by cube, then all used columns
             // must match current schema
             if (factTable.getIdentity().equals(fullTableName)) {
-                List<String> violateColumns = checkAllUsedColumns(cube, factTable,
currentFieldsMap);
+                List<String> violateColumns = checkAllColumnsInCube(cube, factTable,
currentFieldsMap);
                 if (!violateColumns.isEmpty()) {
                     issues.add(format("Column %s used in cube[%s] and model[%s], but changed
in hive", violateColumns, cube.getName(), modelName));
                 }
             }
 
-            // if user reloads a lookup table used by cube, then nearly all changes in schema
are disallowed)
+            // if user reloads a lookup table used by cube, only append column(s) are allowed,
all existing columns
+            // must be the same (except compatible type changes)
             for (TableDesc lookupTable : lookupTables) {
-                if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable,
currentFields)) {
+                if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumnsInTableDesc(lookupTable,
currentFields)) {
                     issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and
model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName));
                 }
             }


Mime
View raw message