kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager
Date Thu, 24 Nov 2016 09:25:19 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 [created] fde30a157


KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fde30a15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fde30a15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fde30a15

Branch: refs/heads/KYLIN-2217-2
Commit: fde30a157dff088b796cc778be34201a4e6e6dd4
Parents: 827205f
Author: Li Yang <liyang@apache.org>
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Thu Nov 24 17:24:57 2016 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/cube/CubeManager.java | 47 +++++++-------
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +++++++++-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java    | 65 +++++++++-----------
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java     |  6 +-
 7 files changed, 93 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..3a4c754 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
         return result;
     }
 
-    public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, DistinctColumnValuesProvider
factTableValueProvider) throws IOException {
+    public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable
inpTable) throws IOException {
         CubeDesc cubeDesc = cubeSeg.getCubeDesc();
         if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
             return null;
 
-        DictionaryManager dictMgr = getDictionaryManager();
         String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-        DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), col, factTableValueProvider,
builderClass);
+        DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(cubeDesc.getModel(),
col, inpTable, builderClass);
 
+        saveDictionaryInfo(cubeSeg, col, dictInfo);
+        return dictInfo;
+    }
+    
+    public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable
inpTable, Dictionary<String> dict) throws IOException {
+        CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+        if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+            return null;
+
+        DictionaryInfo dictInfo = getDictionaryManager().saveDictionary(cubeDesc.getModel(),
col, inpTable, dict);
+        
+        saveDictionaryInfo(cubeSeg, col, dictInfo);
+        return dictInfo;
+    }
+
+    private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, DictionaryInfo dictInfo)
throws IOException {
         if (dictInfo != null) {
             Dictionary<?> dict = dictInfo.getDictionaryObject();
             cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
             cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId()
});
 
-            CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-            cubeBuilder.setToUpdateSegs(cubeSeg);
-            updateCube(cubeBuilder);
+            CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+            update.setToUpdateSegs(cubeSeg);
+            updateCube(update);
         }
-        return dictInfo;
     }
 
     /**
@@ -617,24 +630,6 @@ public class CubeManager implements IRealizationProvider {
         }
     }
 
-    private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
-        List<CubeSegment> existing = cube.getSegments();
-        if (existing.isEmpty()) {
-            return 0;
-        } else {
-            return existing.get(existing.size() - 1).getSourceOffsetEnd();
-        }
-    }
-
-    private long calculateStartDateForAppendSegment(CubeInstance cube) {
-        List<CubeSegment> existing = cube.getSegments();
-        if (existing.isEmpty()) {
-            return cube.getDescriptor().getPartitionDateStart();
-        } else {
-            return existing.get(existing.size() - 1).getDateRangeEnd();
-        }
-    }
-
     private void checkBuildingSegment(CubeInstance cube) {
         int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
         if (cube.getBuildingSegments().size() >= maxBuldingSeg) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -26,9 +26,15 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.DistinctColumnValuesProvider;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.ReadableTable;
+import org.apache.kylin.source.SourceFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,7 +57,8 @@ public class DictionaryGeneratorCLI {
         // dictionary
         for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
             logger.info("Building dictionary for " + col);
-            cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
+            ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, factTableValueProvider);
+            cubeMgr.buildDictionary(cubeSeg, col, inpTable);
         }
 
         // snapshot
@@ -67,4 +74,29 @@ public class DictionaryGeneratorCLI {
             cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity);
         }
     }
+    
+    private static ReadableTable decideInputTable(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider
factTableValueProvider) {
+        KylinConfig config = model.getConfig();
+        DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+        TblColRef srcCol = dictMgr.decideSourceData(model, col);
+        String srcTable = srcCol.getTable();
+        
+        ReadableTable inpTable;
+        if (model.isFactTable(srcTable)) {
+            inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
+        } else {
+            MetadataManager metadataManager = MetadataManager.getInstance(config);
+            TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
+            if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType()))
{
+                TableDesc materializedTbl = new TableDesc();
+                materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
+                materializedTbl.setName(tableDesc.getMaterializedName());
+                inpTable = SourceFactory.createReadableTable(materializedTbl);
+            } else {
+                inpTable = SourceFactory.createReadableTable(tableDesc);
+            }
+        }
+
+        return inpTable;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 10dad2c..413b907 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -170,7 +170,7 @@ public class CubingUtils {
             signature.setLastModifiedTime(System.currentTimeMillis());
             signature.setPath(String.format("streaming_%s_%s", startOffset, endOffset));
             signature.setSize(endOffset - startOffset);
-            DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getTable(), tblColRef.getName(),
tblColRef.getColumnDesc().getZeroBasedIndex(), tblColRef.getDatatype(), signature);
+            DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(),
signature);
             logger.info("writing dictionary for TblColRef:" + tblColRef.toString());
             DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig());
             try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 4fba59a..8526467 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.source.ReadableTable.TableSignature;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -51,6 +52,10 @@ public class DictionaryInfo extends RootPersistentEntity {
     public DictionaryInfo() {
     }
 
+    public DictionaryInfo(ColumnDesc col, String dataType, TableSignature input) {
+        this(col.getTable().getIdentity(), col.getName(), col.getZeroBasedIndex(), dataType,
input);
+    }
+    
     public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex,
String dataType, TableSignature input) {
 
         this.updateRandomUuid();

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index c33cd28..37e4757 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -38,12 +38,10 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.source.ReadableTable;
 import org.apache.kylin.source.ReadableTable.TableSignature;
-import org.apache.kylin.source.SourceFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -273,45 +271,19 @@ public class DictionaryManager {
         }
     }
 
-    public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider
factTableValueProvider) throws IOException {
-        return buildDictionary(model, col, factTableValueProvider, null);
+    public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable
inpTable) throws IOException {
+        return buildDictionary(model, col, inpTable, null);
     }
 
-    public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider
factTableValueProvider, String builderClass) throws IOException {
+    public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable
inpTable, String builderClass) throws IOException {
 
         logger.info("building dictionary for " + col);
 
-        TblColRef srcCol = decideSourceData(model, col);
-        String srcTable = srcCol.getTable();
-        String srcColName = srcCol.getName();
-        int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
-
-        ReadableTable inpTable;
-        if (model.isFactTable(srcTable)) {
-            inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
-        } else {
-            MetadataManager metadataManager = MetadataManager.getInstance(config);
-            TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
-            if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType()))
{
-                TableDesc materializedTbl = new TableDesc();
-                materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
-                materializedTbl.setName(tableDesc.getMaterializedName());
-                inpTable = SourceFactory.createReadableTable(materializedTbl);
-            } else {
-                inpTable = SourceFactory.createReadableTable(tableDesc);
-            }
-        }
-
-        TableSignature inputSig = inpTable.getSignature();
-        if (inputSig == null) // table does not exists
-            return null;
-
-        DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcColName, srcColIdx, col.getDatatype(),
inputSig);
-
-        String dupDict = checkDupByInfo(dictInfo);
-        if (dupDict != null) {
-            logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing
dictionary at " + dupDict);
-            return getDictionaryInfo(dupDict);
+        DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+        String dupInfo = checkDupByInfo(dictInfo);
+        if (dupInfo != null) {
+            logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing
dictionary at " + dupInfo);
+            return getDictionaryInfo(dupInfo);
         }
 
         logger.info("Building dictionary object " + JsonUtil.writeValueAsString(dictInfo));
@@ -333,6 +305,27 @@ public class DictionaryManager {
         return trySaveNewDict(dictionary, dictInfo);
     }
 
+    public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, ReadableTable
inpTable, Dictionary<String> dictionary) throws IOException {
+        DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+        String dupInfo = checkDupByInfo(dictInfo);
+        if (dupInfo != null) {
+            logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing
dictionary at " + dupInfo);
+            return getDictionaryInfo(dupInfo);
+        }
+
+        return trySaveNewDict(dictionary, dictInfo);
+    }
+
+    private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, ReadableTable
inpTable) throws IOException {
+        TblColRef srcCol = decideSourceData(model, col);
+        TableSignature inputSig = inpTable.getSignature();
+        if (inputSig == null) // table does not exists
+            throw new IllegalStateException("Input table does not exist: " + inpTable);
+
+        DictionaryInfo dictInfo = new DictionaryInfo(srcCol.getColumnDesc(), col.getDatatype(),
inputSig);
+        return dictInfo;
+    }
+
     /**
      * Decide a dictionary's source data, leverage PK-FK relationship.
      */

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 6f3b7c9..075ec80 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -119,7 +119,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
             signature.setLastModifiedTime(System.currentTimeMillis());
             signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
 
-            DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(),
lfn.getColumnDesc().getZeroBasedIndex(), "string", signature);
+            DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getColumnDesc(), "string",
signature);
 
             List<String> values = new ArrayList<>();
             values.add("aaa");

http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
index 22ffd26..188a97a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
@@ -65,10 +65,10 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase
{
 
         MockDistinctColumnValuesProvider mockupData = new MockDistinctColumnValuesProvider("A",
"B", "C");
 
-        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+        DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
         System.out.println(JsonUtil.writeValueAsIndentString(info1));
 
-        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+        DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
         System.out.println(JsonUtil.writeValueAsIndentString(info2));
 
         // test check duplicate
@@ -89,7 +89,7 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase {
 
         // test empty dictionary
         MockDistinctColumnValuesProvider mockupEmpty = new MockDistinctColumnValuesProvider();
-        DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty);
+        DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty.getDistinctValuesFor(col));
         System.out.println(JsonUtil.writeValueAsIndentString(info3));
         assertEquals(0, info3.getCardinality());
         assertEquals(0, info3.getDictionaryObject().getSize());


Mime
View raw message