kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject incubator-kylin git commit: KYLIN-839 use dictionary in building snapshot
Date Mon, 29 Jun 2015 02:14:41 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.7-staging c47813f59 -> b97d0e550


KYLIN-839 use dictionary in building snapshot

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b97d0e55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b97d0e55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b97d0e55

Branch: refs/heads/0.7-staging
Commit: b97d0e550739a9ebc22249eca37e025ec2479d36
Parents: c47813f
Author: shaofengshi <shaofengshi@apache.org>
Authored: Sun Jun 28 09:56:32 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Sun Jun 28 09:56:32 2015 +0800

----------------------------------------------------------------------
 .../apache/kylin/dict/lookup/SnapshotTable.java | 61 +++++++++++++++++---
 1 file changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b97d0e55/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index faa4fe6..35cbb84 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -32,6 +32,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionary;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
 import org.apache.kylin.metadata.model.TableDesc;
 
 /**
@@ -44,6 +48,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
     private TableSignature signature;
     @JsonProperty("column_delimeter")
     private String columnDelimeter;
+    @JsonProperty("useDictionary")
+    private Boolean useDictionary;
 
     private ArrayList<String[]> rows;
 
@@ -54,6 +60,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
     SnapshotTable(ReadableTable table) throws IOException {
         this.signature = table.getSignature();
         this.columnDelimeter = table.getColumnDelimeter();
+        this.useDictionary = Boolean.TRUE;
     }
 
     public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException
{
@@ -154,25 +161,61 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
         if (rows.size() > 0) {
             int n = rows.get(0).length;
             out.writeInt(n);
-            for (int i = 0; i < rows.size(); i++) {
-                String[] row = rows.get(i);
-                for (int j = 0; j < n; j++) {
-                    out.writeUTF(row[j]);
+
+            if (this.useDictionary == Boolean.TRUE) {
+                Dictionary<String> dict = buildDictionary();
+                dict.write(out);
+                for (int i = 0; i < rows.size(); i++) {
+                    String[] row = rows.get(i);
+                    for (int j = 0; j < n; j++) {
+                        out.writeInt(dict.getIdFromValue(row[j]));
+                    }
+                }
+
+            } else {
+                for (int i = 0; i < rows.size(); i++) {
+                    String[] row = rows.get(i);
+                    for (int j = 0; j < n; j++) {
+                        out.writeUTF(row[j]);
+                    }
                 }
             }
         }
     }
 
+    Dictionary<String> buildDictionary() {
+        TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new
StringBytesConverter());
+        for (String[] row : rows) {
+            for (String cell : row) {
+                b.addValue(cell);
+            }
+        }
+        TrieDictionary<String> dict = b.build(0);
+        return dict;
+    }
+
     void readData(DataInput in) throws IOException {
         int rowNum = in.readInt();
         rows = new ArrayList<String[]>(rowNum);
         if (rowNum > 0) {
             int n = in.readInt();
-            for (int i = 0; i < rowNum; i++) {
-                String[] row = new String[n];
-                rows.add(row);
-                for (int j = 0; j < n; j++) {
-                    row[j] = in.readUTF();
+            if (this.useDictionary == Boolean.TRUE) {
+                Dictionary<String> dict = new TrieDictionary<String>();
+                dict.readFields(in);
+                for (int i = 0; i < rowNum; i++) {
+                    String[] row = new String[n];
+                    rows.add(row);
+                    for (int j = 0; j < n; j++) {
+                        row[j] = dict.getValueFromId(in.readInt());
+                    }
+                }
+            } else {
+                for (int i = 0; i < rowNum; i++) {
+                    String[] row = new String[n];
+                    rows.add(row);
+                    for (int j = 0; j < n; j++) {
+                        row[j] = in.readUTF();
+                    }
                 }
             }
         }


Mime
View raw message