kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [33/50] [abbrv] incubator-kylin git commit: refactor
Date Sat, 14 Mar 2015 00:01:08 GMT
refactor


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb673625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb673625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb673625

Branch: refs/heads/streaming-localdict
Commit: bb6736258fd9146b9f7fb04745e1ad1990738193
Parents: ab458df
Author: qianhao.zhou <qianzhou@ebay.com>
Authored: Thu Mar 12 18:25:11 2015 +0800
Committer: qianhao.zhou <qianzhou@ebay.com>
Committed: Thu Mar 12 18:25:11 2015 +0800

----------------------------------------------------------------------
 .../index/CompressedValueContainer.java           |  8 ++++++--
 .../invertedindex/model/IIKeyValueCodec.java      | 18 +++++++++++++++++-
 .../storage/hbase/HBaseClientKVIterator.java      |  2 +-
 3 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
index 7a65489..7bd2266 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
@@ -42,8 +42,12 @@ public class CompressedValueContainer implements ColumnValueContainer {
     byte[] compressed;
 
     public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
-        this.valueLen = digest.length(col);
-        this.nValues = digest.getMaxID(col) + 1;
+        this(digest.length(col), digest.getMaxID(col) + 1, cap);
+    }
+
+    public CompressedValueContainer(int valueLen, int nValues, int cap) {
+        this.valueLen = valueLen;
+        this.nValues = nValues;
         this.cap = cap;
         this.size = 0;
         this.uncompressed = null;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 2ebd53d..6955229 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -22,12 +22,14 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.invertedindex.index.ColumnValueContainer;
 import org.apache.kylin.invertedindex.index.CompressedValueContainer;
 import org.apache.kylin.invertedindex.index.Slice;
 import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
 
+import java.io.*;
 import java.util.*;
 
 /**
@@ -137,6 +139,18 @@ public class IIKeyValueCodec implements KeyValueCodec {
 			this.iterator = kvs.iterator();
 		}
 
+        private Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
+            try {
+                final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(),
dictBytes.getOffset(), dictBytes.getLength()));
+                final String type = dataInputStream.readUTF();
+                final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+                dictionary.readFields(dataInputStream);
+                return dictionary;
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
 		private void goToNext() {
 			if (slice != null) { // was not fetched
 				return;
@@ -148,7 +162,9 @@ public class IIKeyValueCodec implements KeyValueCodec {
 				ImmutableBytesWritable k = kv.getKey();
 				ImmutableBytesWritable v = kv.getValue();
 				decodeKey(k);
-//                localDictionaries.put(curCol, kv.getDictionary());
+                final Dictionary<?> dictionary = deserialize(kv.getDictionary());
+                addContainer(curCol, new CompressedValueContainer(dictionary.getSizeOfId(),
(dictionary.getMaxId() - dictionary.getMinId() + 1), (dictionary.getMaxId() - dictionary.getMinId()
+ 1)));
+                localDictionaries.put(curCol, dictionary);
 
 				if (curShard != lastShard
 						|| curSliceTimestamp != lastSliceTimestamp) {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 05d2421..6c1085e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -67,7 +67,7 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable
{
 
         ImmutableBytesWritable key = new ImmutableBytesWritable();
         ImmutableBytesWritable value = new ImmutableBytesWritable();
-        IIRow pair = new IIRow(key, value);
+        IIRow pair = new IIRow(key, value, null);
 
         @Override
         public boolean hasNext() {


Mime
View raw message