kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [09/20] kylin git commit: KYLIN-2304 Only copy latest version dict for global dict
Date Tue, 17 Jan 2017 03:30:26 GMT
KYLIN-2304 Only copy latest version dict for global dict


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb18ac59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb18ac59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb18ac59

Branch: refs/heads/master-hbase1.x
Commit: eb18ac59fa74b3f2f1f0163f7728739a1027a233
Parents: 4b00ec2
Author: kangkaisen <kangkaisen@live.com>
Authored: Sun Dec 18 20:09:03 2016 +0800
Committer: kangkaisen <kangkaisen@163.com>
Committed: Sat Jan 14 19:07:59 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/dict/AppendTrieDictionary.java  | 19 +++++++++++++------
 .../org/apache/kylin/dict/CachedTreeMap.java     |  2 +-
 2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 5029bc4..80403cc 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -1173,13 +1173,15 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T>
{
 
     @Override
     public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig)
throws IOException {
+        //copy appendDict
         Configuration conf = new Configuration();
-        AppendTrieDictionary newDict = new AppendTrieDictionary();
-        newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()),
baseId, maxId, maxValueLength, nValues, bytesConvert);
-        newDict.initDictSliceMap((CachedTreeMap)dictSliceMap);
-        logger.info("Copy AppendDict from {} to {}", this.baseDir, newDict.baseDir);
-        Path srcPath = new Path(this.baseDir);
-        Path dstPath = new Path(newDict.baseDir);
+
+        Path base = new Path(baseDir);
+        FileSystem srcFs = FileSystem.get(base.toUri(), conf);
+        Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base);
+        Path dstPath = new Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()));
+        logger.info("Copy appendDict from {} to {}", srcPath, dstPath);
+
         FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf);
         if (dstFs.exists(dstPath)) {
             logger.info("Delete existing AppendDict {}", dstPath);
@@ -1187,6 +1189,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T>
{
         }
         FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(),
conf), dstPath, false, true, conf);
 
+        // init new AppendTrieDictionary
+        AppendTrieDictionary newDict = new AppendTrieDictionary();
+        newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()),
baseId, maxId, maxValueLength, nValues, bytesConverter);
+        newDict.initDictSliceMap((CachedTreeMap) dictSliceMap);
+
         return newDict;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 3be4fd7..cc23261 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -225,7 +225,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends
Writable> ext
         return getLatestVersion(conf, fs, baseDir).toUri().getPath();
     }
 
-    private static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir)
throws IOException {
+    public static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir)
throws IOException {
         String[] versions = listAllVersions(fs, baseDir);
         if (versions.length > 0) {
             return new Path(versions[versions.length - 1]);


Mime
View raw message