kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [3/3] kylin git commit: KYLIN-1492 Custom dimension encoding support
Date Mon, 14 Mar 2016 08:44:10 GMT
KYLIN-1492 Custom dimension encoding support


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75027ed6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75027ed6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75027ed6

Branch: refs/heads/master
Commit: 75027ed649c4dfba02e80004495875a31dc73702
Parents: 959c6dd
Author: Li, Yang <liyang@apache.org>
Authored: Fri Feb 26 14:43:59 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Mon Mar 14 16:42:33 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/util/Dictionary.java    | 232 -------------------
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |  21 +-
 .../kylin/cube/common/RowKeySplitter.java       |   3 +-
 .../kylin/cube/gridtable/CubeCodeSystem.java    | 141 ++---------
 .../kylin/cube/gridtable/CubeGridTable.java     |  42 +---
 .../gridtable/CuboidToGridTableMapping.java     |  11 +
 .../kylin/cube/gridtable/FixLenSerializer.java  | 110 ---------
 .../cube/gridtable/TrimmedCubeCodeSystem.java   | 148 +++++++-----
 .../inmemcubing/AbstractInMemCubeBuilder.java   |   2 +-
 .../cube/inmemcubing/DoggedCubeBuilder.java     |   2 +-
 .../cube/inmemcubing/InMemCubeBuilder.java      |   2 +-
 .../InMemCubeBuilderInputConverter.java         |   3 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java    |   4 +-
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |  80 +++++++
 .../org/apache/kylin/cube/kv/RowConstants.java  |   2 -
 .../apache/kylin/cube/kv/RowKeyColumnIO.java    | 149 ++----------
 .../org/apache/kylin/cube/kv/RowKeyDecoder.java |   4 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |   4 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  12 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |   2 +-
 .../kylin/gridtable/DefaultGTComparator.java    |  10 +-
 .../kylin/cube/DictionaryManagerTest.java       |   2 +-
 .../DoggedCubeBuilderStressTest.java            |   2 +-
 .../cube/inmemcubing/DoggedCubeBuilderTest.java |   2 +-
 .../cube/inmemcubing/InMemCubeBuilderTest.java  |   2 +-
 .../kylin/gridtable/DictGridTableTest.java      |  18 +-
 .../apache/kylin/dict/DateStrDictionary.java    |   2 +-
 .../org/apache/kylin/dict/DictCodeSystem.java   |   4 +-
 .../apache/kylin/dict/DictionaryGenerator.java  |   2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |   2 +-
 .../kylin/dict/DictionaryInfoSerializer.java    |   2 +-
 .../apache/kylin/dict/DictionaryManager.java    |   2 +-
 .../apache/kylin/dict/DictionarySerializer.java |   2 +-
 .../org/apache/kylin/dict/IDictionaryAware.java |  34 ---
 .../dict/MultipleDictionaryValueEnumerator.java |   2 +-
 .../apache/kylin/dict/TimeStrDictionary.java    |   2 +-
 .../org/apache/kylin/dict/TrieDictionary.java   |   2 +-
 .../dict/TupleFilterFunctionTransformer.java    |  16 +-
 .../apache/kylin/dict/lookup/SnapshotTable.java |   2 +-
 .../apache/kylin/dict/NumberDictionaryTest.java |   2 +-
 .../kylin/dict/TimeStrDictionaryTests.java      |   4 +-
 .../org/apache/kylin/dimension/Dictionary.java  | 231 ++++++++++++++++++
 .../kylin/dimension/DictionaryDimEnc.java       | 137 +++++++++++
 .../kylin/dimension/DimensionEncoding.java      |  62 +++++
 .../apache/kylin/dimension/FixedLenDimEnc.java  | 136 +++++++++++
 .../kylin/dimension/IDimensionEncodingMap.java  |  40 ++++
 .../apache/kylin/measure/MeasureIngester.java   |   2 +-
 .../org/apache/kylin/measure/MeasureType.java   |   2 +-
 .../kylin/measure/basic/BigDecimalIngester.java |   2 +-
 .../kylin/measure/basic/DoubleIngester.java     |   2 +-
 .../kylin/measure/basic/LongIngester.java       |   2 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |   2 +-
 .../ExtendedColumnMeasureType.java              |   2 +-
 .../kylin/measure/hllc/HLLCMeasureType.java     |   2 +-
 .../kylin/measure/topn/TopNMeasureType.java     |   2 +-
 .../storage/translate/ColumnValueRange.java     |   2 +-
 .../storage/translate/ColumnValueRangeTest.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   4 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |   2 +-
 .../engine/mr/steps/InMemCuboidMapper.java      |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |   2 +-
 .../engine/mr/steps/MergeCuboidMapperTest.java  |   2 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   6 +-
 .../streaming/OneOffStreamingBuilder.java       |   2 +-
 .../engine/streaming/StreamingBatchBuilder.java |   2 +-
 .../streaming/cube/StreamingCubeBuilder.java    |   2 +-
 .../apache/kylin/invertedindex/IISegment.java   |   7 +-
 .../index/CompressedValueContainer.java         |   2 +-
 .../invertedindex/index/RawTableRecord.java     |   4 +-
 .../apache/kylin/invertedindex/index/Slice.java |   2 +-
 .../kylin/invertedindex/index/SliceBuilder.java |   2 +-
 .../kylin/invertedindex/index/TableRecord.java  |   2 +-
 .../invertedindex/index/TableRecordInfo.java    |   2 +-
 .../invertedindex/model/IIKeyValueCodec.java    |   2 +-
 .../invertedindex/util/IIDictionaryBuilder.java |   2 +-
 .../kylin/invertedindex/IIInstanceTest.java     |   2 +-
 .../invertedindex/InvertedIndexLocalTest.java   |   2 +-
 .../hbase/common/coprocessor/AggrKey.java       |   4 +-
 .../common/coprocessor/CoprocessorFilter.java   |   6 +-
 .../common/coprocessor/CoprocessorRowType.java  |   2 +-
 .../common/coprocessor/FilterDecorator.java     |  18 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v1/CubeTupleConverter.java       |   2 +-
 .../coprocessor/observer/ObserverEnabler.java   |   2 +-
 .../v1/coprocessor/observer/ObserverTuple.java  |   2 +-
 .../hbase/cube/v2/CubeSegmentScanner.java       |   4 +-
 .../hbase/cube/v2/CubeTupleConverter.java       |   2 +-
 .../endpoint/BitMapFilterEvaluator.java         |   2 +-
 .../endpoint/ClearTextDictionary.java           |  31 +--
 .../ii/coprocessor/endpoint/IIEndpoint.java     |  21 +-
 .../coprocessor/endpoint/LocalDictionary.java   |  31 ++-
 .../endpoint/BitMapFilterEvaluatorTest.java     |   2 +-
 93 files changed, 1026 insertions(+), 878 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
deleted file mode 100644
index 6d3fa62..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
-
-import org.apache.kylin.common.persistence.Writable;
-
-/**
- * A bi-way dictionary that maps from dimension/column values to IDs and vice
- * versa. By storing IDs instead of real values, the size of cube is
- * significantly reduced.
- * 
- * - IDs are smallest integers possible for the cardinality of a column, for the
- * purpose of minimal storage space - IDs preserve ordering of values, such that
- * range query can be applied to IDs directly
- * 
- * A dictionary once built, is immutable. This allows optimal memory footprint
- * by e.g. flatten the Trie structure into a byte array, replacing node pointers
- * with array offsets.
- * 
- * @author yangli9
- */
-@SuppressWarnings("serial")
-abstract public class Dictionary<T> implements Writable, Serializable {
-
-    public static final byte NULL = (byte) 0xff;
-
-    // ID with all bit-1 (0xff e.g.) reserved for NULL value
-    public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
-
-    abstract public int getMinId();
-
-    abstract public int getMaxId();
-
-    public int getSize() {
-        return getMaxId() - getMinId() + 1;
-    }
-
-    /**
-     * @return the size of an ID in bytes, determined by the cardinality of column
-     */
-    abstract public int getSizeOfId();
-
-    /**
-     * @return the (maximum) size of value in bytes, determined by the longest value
-     */
-    abstract public int getSizeOfValue();
-
-    /**
-     * @return true if each entry of this dict is contained by the dict in param
-     */
-    abstract public boolean contains(Dictionary<?> another);
-
-    /**
-     * Convenient form of <code>getIdFromValue(value, 0)</code>
-     */
-    final public int getIdFromValue(T value) throws IllegalArgumentException {
-        return getIdFromValue(value, 0);
-    }
-
-    /**
-     * Returns the ID integer of given value. In case of not found
-     * <p>
-     * - if roundingFlag=0, throw IllegalArgumentException; <br>
-     * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
-     * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
-     * <p>
-     * The implementation often has cache, thus faster than the byte[] version getIdFromValueBytes()
-     * 
-     * @throws IllegalArgumentException
-     *             if value is not found in dictionary and rounding is off;
-     *             or if rounding cannot find a smaller or bigger ID
-     */
-    final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
-        if (isNullObjectForm(value))
-            return nullId();
-        else
-            return getIdFromValueImpl(value, roundingFlag);
-    }
-
-    final public boolean containsValue(T value) throws IllegalArgumentException {
-        if (isNullObjectForm(value)) {
-            return true;
-        } else {
-            try {
-                //if no key found, it will throw exception
-                getIdFromValueImpl(value, 0);
-            } catch (IllegalArgumentException e) {
-                return false;
-            }
-            return true;
-        }
-    }
-
-    protected boolean isNullObjectForm(T value) {
-        return value == null;
-    }
-
-    abstract protected int getIdFromValueImpl(T value, int roundingFlag);
-
-    /**
-     * @return the value corresponds to the given ID
-     * @throws IllegalArgumentException
-     *             if ID is not found in dictionary
-     */
-    final public T getValueFromId(int id) throws IllegalArgumentException {
-        if (isNullId(id))
-            return null;
-        else
-            return getValueFromIdImpl(id);
-    }
-
-    abstract protected T getValueFromIdImpl(int id);
-
-    /**
-     * Convenient form of
-     * <code>getIdFromValueBytes(value, offset, len, 0)</code>
-     */
-    final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException {
-        return getIdFromValueBytes(value, offset, len, 0);
-    }
-
-    /**
-     * A lower level API, return ID integer from raw value bytes. In case of not found 
-     * <p>
-     * - if roundingFlag=0, throw IllegalArgumentException; <br>
-     * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
-     * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
-     * <p>
-     * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
-     * 
-     * @throws IllegalArgumentException
-     *             if value is not found in dictionary and rounding is off;
-     *             or if rounding cannot find a smaller or bigger ID
-     */
-    final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
-        if (isNullByteForm(value, offset, len))
-            return nullId();
-        else {
-            int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
-            if (id < 0)
-                throw new IllegalArgumentException("Value not exists!");
-            return id;
-        }
-    }
-
-    protected boolean isNullByteForm(byte[] value, int offset, int len) {
-        return value == null;
-    }
-
-    abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
-
-    final public byte[] getValueBytesFromId(int id) {
-        if (isNullId(id))
-            return BytesUtil.EMPTY_BYTE_ARRAY;
-        else
-            return getValueBytesFromIdImpl(id);
-    }
-
-    abstract protected byte[] getValueBytesFromIdImpl(int id);
-
-    /**
-     * A lower level API, get byte values from ID, return the number of bytes
-     * written. Bypassing the cache layer, this could be significantly slower
-     * than getIdFromValue(T value).
-     *
-     * @return size of value bytes, 0 if empty string, -1 if null
-     *
-     * @throws IllegalArgumentException
-     *             if ID is not found in dictionary
-     */
-    final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
-        if (isNullId(id))
-            return -1;
-        else
-            return getValueBytesFromIdImpl(id, returnValue, offset);
-    }
-
-    abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
-
-    abstract public void dump(PrintStream out);
-
-    public int nullId() {
-        return NULL_ID[getSizeOfId()];
-    }
-
-    public boolean isNullId(int id) {
-        int nullId = NULL_ID[getSizeOfId()];
-        return (nullId & id) == nullId;
-    }
-
-    /** utility that converts a dictionary ID to string, preserving order */
-    public static String dictIdToString(byte[] idBytes, int offset, int length) {
-        try {
-            return new String(idBytes, offset, length, "ISO-8859-1");
-        } catch (UnsupportedEncodingException e) {
-            // never happen
-            return null;
-        }
-    }
-
-    /** the reverse of dictIdToString(), returns integer ID */
-    public static int stringToDictId(String str) {
-        try {
-            byte[] bytes = str.getBytes("ISO-8859-1");
-            return BytesUtil.readUnsigned(bytes, 0, bytes.length);
-        } catch (UnsupportedEncodingException e) {
-            // never happen
-            return 0;
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4951ce6..f53a1f0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -32,7 +32,6 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.restclient.Broadcaster;
 import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
@@ -42,6 +41,7 @@ import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 5b61c10..ead6a41 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -25,12 +25,12 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ShardingHash;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.IDictionaryAware;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -45,7 +45,7 @@ import java.util.TimeZone;
 import java.util.concurrent.ConcurrentHashMap;
 
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, IRealizationSegment {
+public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment {
 
     @JsonBackReference
     private CubeInstance cubeInstance;
@@ -277,20 +277,13 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
         return result;
     }
 
-    @Override
-    public int getColumnLength(TblColRef col) {
-        Dictionary<?> dict = getDictionary(col);
-        if (dict == null) {
-            return this.getCubeDesc().getRowkey().getColumnLength(col);
-        } else {
-            return dict.getSizeOfId();
-        }
-    }
-
-    @Override
     public Dictionary<String> getDictionary(TblColRef col) {
         return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
     }
+    
+    public CubeDimEncMap getDimensionEncodingMap() {
+        return new CubeDimEncMap(this);
+    }
 
     public void validate() {
         if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned() && dateRangeStart >= dateRangeEnd)

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 56247bc..a8be987 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -22,6 +22,7 @@ import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.SplittedBytes;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.kv.RowKeyColumnIO;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -58,7 +59,7 @@ public class RowKeySplitter {
     public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
         this.enableSharding = cubeSeg.isEnableSharding();
         this.cubeDesc = cubeSeg.getCubeDesc();
-        this.colIO = new RowKeyColumnIO(cubeSeg);
+        this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg));
 
         this.splitBuffers = new SplittedBytes[splitLen];
         for (int i = 0; i < splitLen; i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f15d3f5..e0dc4dd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -22,9 +22,10 @@ import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.Map;
 
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.DefaultGTComparator;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.IGTCodeSystem;
@@ -32,8 +33,6 @@ import org.apache.kylin.gridtable.IGTComparator;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
-import com.google.common.collect.Maps;
-
 /**
  * defines how column values will be encoded to/ decoded from GTRecord 
  * 
@@ -43,63 +42,44 @@ import com.google.common.collect.Maps;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class CubeCodeSystem implements IGTCodeSystem {
 
-    // ============================================================================
-
     private GTInfo info;
 
-    private Map<Integer, Dictionary> dictionaryMap; // column index ==> dictionary of column
-    private Map<Integer, Integer> fixLenMap; // column index ==> fixed length of column
-    private Map<Integer, Integer> dependentMetricsMap;
+    private DimensionEncoding[] dimEncs;
     private DataTypeSerializer[] serializers;
     private IGTComparator comparator;
+    private Map<Integer, Integer> dependentMetricsMap;
 
-    public CubeCodeSystem(Map<Integer, Dictionary> dictionaryMap) {
-        this(dictionaryMap, Collections.<Integer, Integer> emptyMap(), Collections.<Integer, Integer> emptyMap());
+    public CubeCodeSystem(DimensionEncoding[] dimEncs) {
+        this(dimEncs, Collections.<Integer, Integer> emptyMap());
     }
 
-    public CubeCodeSystem(Map<Integer, Dictionary> dictionaryMap, Map<Integer, Integer> fixLenMap, Map<Integer, Integer> dependentMetricsMap) {
-        this.dictionaryMap = dictionaryMap;
-        this.fixLenMap = fixLenMap;
+    public CubeCodeSystem(DimensionEncoding[] dimEncs, Map<Integer, Integer> dependentMetricsMap) {
+        this.dimEncs = dimEncs;
+        this.comparator = new DefaultGTComparator();
         this.dependentMetricsMap = dependentMetricsMap;
     }
 
     public TrimmedCubeCodeSystem trimForCoprocessor() {
-        Map<Integer,Integer> dictSizes = Maps.newHashMap();
-        Map<Integer,Integer> fixedLengthSizes = Maps.newHashMap();
-
-        for (int i = 0; i < serializers.length; i++) {
-            if (serializers[i] instanceof DictionarySerializer) {
-                dictSizes.put(i,serializers[i].maxLength());
-            } else if(serializers[i] instanceof  FixLenSerializer) {
-                fixedLengthSizes.put(i,serializers[i].maxLength());
-            }
-        }
-
-        return new TrimmedCubeCodeSystem(dependentMetricsMap,dictSizes,fixedLengthSizes);
+        return new TrimmedCubeCodeSystem(dimEncs, dependentMetricsMap);
     }
 
     @Override
     public void init(GTInfo info) {
         this.info = info;
 
-        serializers = new DataTypeSerializer[info.getColumnCount()];
-        for (int i = 0; i < info.getColumnCount(); i++) {
-            // dimension with dictionary
-            if (dictionaryMap.get(i) != null) {
-                serializers[i] = new DictionarySerializer(dictionaryMap.get(i));
-            }
-            // dimension of fixed length
-            else if (fixLenMap.get(i) != null) {
-                serializers[i] = new FixLenSerializer(fixLenMap.get(i));
+        this.serializers = new DataTypeSerializer[info.getColumnCount()];
+        for (int i = 0; i < serializers.length; i++) {
+            DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null;
+
+            // for dimensions
+            if (dimEnc != null) {
+                serializers[i] = dimEnc.asDataTypeSerializer();
             }
-            // metrics
+            // for measures
             else {
                 serializers[i] = DataTypeSerializer.create(info.getColumnType(i));
             }
         }
-
-        //when changing this, also take care of TrimmedCubeCodeSystem.init
-        this.comparator = new DefaultGTComparator();
     }
 
     @Override
@@ -126,10 +106,14 @@ public class CubeCodeSystem implements IGTCodeSystem {
     public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
         DataTypeSerializer serializer = serializers[col];
         if (serializer instanceof DictionarySerializer) {
-            ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
+            DictionaryDimEnc dictEnc = ((DictionaryDimEnc) dimEncs[col]);
+            if (dictEnc.getRoundingFlag() != roundingFlag) {
+                serializer = dictEnc.copy(roundingFlag).asDataTypeSerializer();
+            }
+            serializer.serialize(value, buf);
         } else {
             if (value instanceof String) {
-                // for dimensions mostly, measures are converted by MeasureIngestor before reaching this point
+                // for dimensions; measures are converted by MeasureIngestor before reaching this point
                 value = serializer.valueOf((String) value);
             }
             serializer.serialize(value, buf);
@@ -169,79 +153,4 @@ public class CubeCodeSystem implements IGTCodeSystem {
         return result;
     }
 
-    static class TrimmedDictionarySerializer extends DataTypeSerializer {
-
-        final int fieldSize;
-
-        public TrimmedDictionarySerializer(int fieldSize) {
-            this.fieldSize = fieldSize;
-        }
-
-        @Override
-        public int peekLength(ByteBuffer in) {
-            return fieldSize;
-        }
-
-        @Override
-        public int maxLength() {
-            return fieldSize;
-        }
-
-        @Override
-        public int getStorageBytesEstimate() {
-            return fieldSize;
-        }
-
-        @Override
-        public void serialize(Object value, ByteBuffer out) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public Object deserialize(ByteBuffer in) {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    static class DictionarySerializer extends DataTypeSerializer {
-        private Dictionary dictionary;
-
-        DictionarySerializer(Dictionary dictionary) {
-            this.dictionary = dictionary;
-        }
-
-        public void serializeWithRounding(Object value, int roundingFlag, ByteBuffer buf) {
-            int id = dictionary.getIdFromValue(value, roundingFlag);
-            BytesUtil.writeUnsigned(id, dictionary.getSizeOfId(), buf);
-        }
-
-        @Override
-        public void serialize(Object value, ByteBuffer buf) {
-            int id = dictionary.getIdFromValue(value);
-            BytesUtil.writeUnsigned(id, dictionary.getSizeOfId(), buf);
-        }
-
-        @Override
-        public Object deserialize(ByteBuffer in) {
-            int id = BytesUtil.readUnsigned(in, dictionary.getSizeOfId());
-            return dictionary.getValueFromId(id);
-        }
-
-        @Override
-        public int peekLength(ByteBuffer in) {
-            return dictionary.getSizeOfId();
-        }
-
-        @Override
-        public int maxLength() {
-            return dictionary.getSizeOfId();
-        }
-
-        @Override
-        public int getStorageBytesEstimate() {
-            return dictionary.getSizeOfId();
-        }
-
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 47885f7..62ba6eb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -21,17 +21,18 @@ package org.apache.kylin.cube.gridtable;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import com.google.common.collect.Maps;
 
-@SuppressWarnings("rawtypes")
 public class CubeGridTable {
 
     public static Map<TblColRef, Dictionary<String>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) {
@@ -52,44 +53,21 @@ public class CubeGridTable {
     }
 
     public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
-        Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
         Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
-        for (TblColRef dim : cuboid.getColumns()) {
-            if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
-                Dictionary dict = dictionaryMap.get(dim);
-                if (dict == null) {
-                    throw new RuntimeException("Dictionary for " + dim + " is not found");
-                }
-            }
-        }
-
-        return newGTInfo(cubeSeg.getCubeDesc(), cuboidId, dictionaryMap);
+        return newGTInfo(cuboid, new CubeDimEncMap(cubeSeg));
     }
 
     public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary<String>> dictionaryMap) {
         Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
+        return newGTInfo(cuboid, new CubeDimEncMap(cubeDesc, dictionaryMap));
+    }
+    
+    public static GTInfo newGTInfo(Cuboid cuboid, IDimensionEncodingMap dimEncMap) {
         CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(cuboid);
 
-        Map<Integer, Dictionary> dictionaryByColIdx = Maps.newHashMap();
-        Map<Integer, Integer> fixLenByColIdx = Maps.newHashMap();
-
-        for (TblColRef dim : cuboid.getColumns()) {
-            int colIndex = mapping.getIndexOf(dim);
-            if (cubeDesc.getRowkey().isUseDictionary(dim)) {
-                Dictionary dict = dictionaryMap.get(dim);
-                dictionaryByColIdx.put(colIndex, dict);
-            } else {
-                int len = cubeDesc.getRowkey().getColumnLength(dim);
-                if (len == 0)
-                    throw new IllegalStateException();
-
-                fixLenByColIdx.put(colIndex, len);
-            }
-        }
-
         GTInfo.Builder builder = GTInfo.builder();
-        builder.setTableName("Cuboid " + cuboidId);
-        builder.setCodeSystem(new CubeCodeSystem(dictionaryByColIdx, fixLenByColIdx, mapping.getDependentMetricsMap()));
+        builder.setTableName("Cuboid " + cuboid.getId());
+        builder.setCodeSystem(new CubeCodeSystem(mapping.getDimensionEncodings(dimEncMap), mapping.getDependentMetricsMap()));
         builder.setColumns(mapping.getDataTypes());
         builder.setPrimaryKey(mapping.getPrimaryKey());
         builder.enableColumnBlock(mapping.getColumnBlocks());

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index aad37fd..75ab140 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -28,6 +28,8 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.HBaseColumnDesc;
 import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -152,6 +154,15 @@ public class CuboidToGridTableMapping {
         return cuboid.getColumns();
     }
 
+    public DimensionEncoding[] getDimensionEncodings(IDimensionEncodingMap dimEncMap) {
+        List<TblColRef> dims = cuboid.getColumns();
+        DimensionEncoding[] dimEncs = new DimensionEncoding[dims.size()];
+        for (int i = 0; i < dimEncs.length; i++) {
+            dimEncs[i] = dimEncMap.get(dims.get(i));
+        }
+        return dimEncs;
+    }
+
     public Map<Integer, Integer> getDependentMetricsMap() {
         Map<Integer, Integer> result = Maps.newHashMap();
         List<MeasureDesc> measures = cuboid.getCubeDesc().getMeasures();

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
deleted file mode 100644
index 24c4a19..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FixLenSerializer extends DataTypeSerializer {
-
-    private static Logger logger = LoggerFactory.getLogger(FixLenSerializer.class);
-
-    // be thread-safe and avoid repeated obj creation
-    private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
-
-    private int fixLen;
-    transient int avoidVerbose = 0;
-
-    FixLenSerializer(int fixLen) {
-        this.fixLen = fixLen;
-    }
-
-    private byte[] currentBuf() {
-        byte[] buf = current.get();
-        if (buf == null) {
-            buf = new byte[fixLen];
-            current.set(buf);
-        }
-        return buf;
-    }
-
-    @Override
-    public void serialize(Object value, ByteBuffer out) {
-        byte[] buf = currentBuf();
-        if (value == null) {
-            Arrays.fill(buf, Dictionary.NULL);
-            out.put(buf);
-        } else {
-            byte[] bytes = Bytes.toBytes(value.toString());
-            if (bytes.length > fixLen) {
-                if (avoidVerbose++ % 10000 == 0) {
-                    logger.warn("Expect at most " + fixLen + " bytes, but got " + bytes.length + ", will truncate, value string: " + value.toString() + " times:" + avoidVerbose);
-                }
-            }
-            out.put(bytes, 0, Math.min(bytes.length, fixLen));
-            for (int i = bytes.length; i < fixLen; i++) {
-                out.put(RowConstants.ROWKEY_PLACE_HOLDER_BYTE);
-            }
-        }
-    }
-
-    @Override
-    public Object deserialize(ByteBuffer in) {
-        byte[] buf = currentBuf();
-        in.get(buf);
-
-        int tail = fixLen;
-        while (tail > 0 && (buf[tail - 1] == RowConstants.ROWKEY_PLACE_HOLDER_BYTE || buf[tail - 1] == Dictionary.NULL)) {
-            tail--;
-        }
-
-        if (tail == 0) {
-            return buf[0] == Dictionary.NULL ? null : "";
-        }
-
-        return Bytes.toString(buf, 0, tail);
-    }
-
-    @Override
-    public int peekLength(ByteBuffer in) {
-        return fixLen;
-    }
-
-    @Override
-    public int maxLength() {
-        return fixLen;
-    }
-
-    @Override
-    public int getStorageBytesEstimate() {
-        return fixLen;
-    }
-
-    @Override
-    public Object valueOf(String str) {
-        return str;
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index 6048ba0..e5169d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.DefaultGTComparator;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.IGTCodeSystem;
@@ -35,44 +36,44 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
 import com.google.common.collect.Maps;
 
+/**
+ * A limited code system where dimension value ser/des is disabled.
+ * Used inside coprocessor only. Because dictionary is not available. 
+ */
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class TrimmedCubeCodeSystem implements IGTCodeSystem {
 
-    private Map<Integer, Integer> dependentMetricsMap;
-    private Map<Integer, Integer> dictSizes;
-    private Map<Integer, Integer> fixedLengthSize;
+    private GTInfo info;
 
-    private transient GTInfo info;
-    private transient DataTypeSerializer[] serializers;
-    private transient IGTComparator comparator;
+    private DimensionEncoding[] dimEncs;
+    private DataTypeSerializer[] serializers;
+    private IGTComparator comparator;
+    private Map<Integer, Integer> dependentMetricsMap;
 
-    public TrimmedCubeCodeSystem(Map<Integer, Integer> dependentMetricsMap, Map<Integer, Integer> dictSizes, Map<Integer, Integer> fixedLengthSize) {
+    public TrimmedCubeCodeSystem(DimensionEncoding[] dimEncs, Map<Integer, Integer> dependentMetricsMap) {
+        this.dimEncs = dimEncs;
+        this.comparator = new DefaultGTComparator();
         this.dependentMetricsMap = dependentMetricsMap;
-        this.dictSizes = dictSizes;
-        this.fixedLengthSize = fixedLengthSize;
     }
 
     @Override
     public void init(GTInfo info) {
         this.info = info;
 
-        serializers = new DataTypeSerializer[info.getColumnCount()];
-        for (int i = 0; i < info.getColumnCount(); i++) {
-            // dimension with dictionary
-            if (dictSizes.get(i) != null) {
-                serializers[i] = new CubeCodeSystem.TrimmedDictionarySerializer(dictSizes.get(i));
-            }
-            // dimension of fixed length
-            else if (fixedLengthSize.get(i) != null) {
-                serializers[i] = new FixLenSerializer(fixedLengthSize.get(i));
+        this.serializers = new DataTypeSerializer[info.getColumnCount()];
+        for (int i = 0; i < serializers.length; i++) {
+            DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null;
+
+            // for dimensions
+            if (dimEnc != null) {
+                // use trimmed serializer cause no dictionary in coprocessor
+                serializers[i] = new TrimmedDimensionSerializer(dimEnc.getLengthOfEncoding());
             }
-            // metrics
+            // for measures
             else {
                 serializers[i] = DataTypeSerializer.create(info.getColumnType(i));
             }
         }
-
-        this.comparator = new DefaultGTComparator();
     }
 
     @Override
@@ -98,11 +99,6 @@ public class TrimmedCubeCodeSystem implements IGTCodeSystem {
     @Override
     public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
         DataTypeSerializer serializer = serializers[col];
-
-        //        if (((value instanceof String) && !(serializer instanceof StringSerializer || serializer instanceof CubeCodeSystem.FixLenSerializer))) {
-        //            value = serializer.valueOf((String) value);
-        //        }
-
         serializer.serialize(value, buf);
     }
 
@@ -149,49 +145,95 @@ public class TrimmedCubeCodeSystem implements IGTCodeSystem {
                 BytesUtil.writeVInt(x.getValue(), out);
             }
 
-            BytesUtil.writeVInt(value.dictSizes.size(), out);
-            for (Map.Entry<Integer, Integer> x : value.dictSizes.entrySet()) {
-                BytesUtil.writeVInt(x.getKey(), out);
-                BytesUtil.writeVInt(x.getValue(), out);
-            }
-
-            BytesUtil.writeVInt(value.fixedLengthSize.size(), out);
-            for (Map.Entry<Integer, Integer> x : value.fixedLengthSize.entrySet()) {
-                BytesUtil.writeVInt(x.getKey(), out);
-                BytesUtil.writeVInt(x.getValue(), out);
+            BytesUtil.writeVInt(value.dimEncs.length, out);
+            for (int i = 0; i < value.dimEncs.length; i++) {
+                DimensionEncoding enc = value.dimEncs[i];
+                BytesUtil.writeVInt(enc == null ? 0 : enc.getLengthOfEncoding(), out);
             }
         }
 
         @Override
         public TrimmedCubeCodeSystem deserialize(ByteBuffer in) {
             Map<Integer, Integer> dependentMetricsMap = Maps.newHashMap();
-            Map<Integer, Integer> dictSizes = Maps.newHashMap();
-            Map<Integer, Integer> fixedLengthSize = Maps.newHashMap();
-
-            int size = 0;
 
-            size = BytesUtil.readVInt(in);
+            int size = BytesUtil.readVInt(in);
             for (int i = 0; i < size; ++i) {
                 int key = BytesUtil.readVInt(in);
                 int value = BytesUtil.readVInt(in);
                 dependentMetricsMap.put(key, value);
             }
 
-            size = BytesUtil.readVInt(in);
-            for (int i = 0; i < size; ++i) {
-                int key = BytesUtil.readVInt(in);
-                int value = BytesUtil.readVInt(in);
-                dictSizes.put(key, value);
+            DimensionEncoding[] dimEncs = new DimensionEncoding[BytesUtil.readVInt(in)];
+            for (int i = 0; i < dimEncs.length; i++) {
+                int fixedLen = BytesUtil.readVInt(in);
+                if (fixedLen > 0)
+                    dimEncs[i] = new TrimmedDimEnc(fixedLen);
             }
 
-            size = BytesUtil.readVInt(in);
-            for (int i = 0; i < size; ++i) {
-                int key = BytesUtil.readVInt(in);
-                int value = BytesUtil.readVInt(in);
-                fixedLengthSize.put(key, value);
-            }
-            return new TrimmedCubeCodeSystem(dependentMetricsMap, dictSizes, fixedLengthSize);
+            return new TrimmedCubeCodeSystem(dimEncs, dependentMetricsMap);
         }
     };
 
+    static class TrimmedDimEnc extends DimensionEncoding {
+        final int fixedLen;
+
+        TrimmedDimEnc(int fixedLen) {
+            this.fixedLen = fixedLen;
+        }
+        
+        @Override
+        public int getLengthOfEncoding() {
+            return fixedLen;
+        }
+
+        @Override
+        public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String decode(byte[] bytes, int offset, int len) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public DataTypeSerializer<Object> asDataTypeSerializer() {
+            throw new UnsupportedOperationException();
+        }
+    }
+    
+    static class TrimmedDimensionSerializer extends DataTypeSerializer<Object> {
+
+        final int fixedLen;
+
+        public TrimmedDimensionSerializer(int fixedLen) {
+            this.fixedLen = fixedLen;
+        }
+
+        @Override
+        public int peekLength(ByteBuffer in) {
+            return fixedLen;
+        }
+
+        @Override
+        public int maxLength() {
+            return fixedLen;
+        }
+
+        @Override
+        public int getStorageBytesEstimate() {
+            return fixedLen;
+        }
+
+        @Override
+        public void serialize(Object value, ByteBuffer out) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Object deserialize(ByteBuffer in) {
+            throw new UnsupportedOperationException();
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 335a769..c567c9e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -21,8 +21,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.GridTable;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index af6ef82..c210bf9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -30,10 +30,10 @@ import java.util.concurrent.ConcurrentNavigableMap;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index c270d3f..ee5a757 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.kylin.measure.topn.Counter;
 import org.apache.kylin.measure.topn.TopNCounter;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
@@ -39,6 +38,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.gridtable.CubeGridTable;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTAggregateScanner;
 import org.apache.kylin.gridtable.GTBuilder;
 import org.apache.kylin.gridtable.GTInfo;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index a1eb6da..4c62bc6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -22,10 +22,11 @@ import org.apache.kylin.common.util.Bytes;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.kylin.common.util.Dictionary;
 import com.google.common.collect.Lists;
+
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.measure.MeasureIngester;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index 62432f7..37b33aa 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -21,10 +21,10 @@ package org.apache.kylin.cube.kv;
 import java.util.Map;
 
 import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
 public abstract class AbstractRowKeyEncoder {
 
     protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class);
-    public static final byte DEFAULT_BLANK_BYTE = Dictionary.NULL;
+    public static final byte DEFAULT_BLANK_BYTE = DimensionEncoding.NULL;
 
     protected byte blankByte = DEFAULT_BLANK_BYTE;
     protected final CubeSegment cubeSeg;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
new file mode 100644
index 0000000..e3c81f1
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.kv;
+
+import java.util.Map;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyColDesc;
+import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.FixedLenDimEnc;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Maps;
+
+public class CubeDimEncMap implements IDimensionEncodingMap {
+
+    final private CubeDesc cubeDesc;
+    final private CubeSegment seg;
+    final private Map<TblColRef, Dictionary<String>> dictionaryMap;
+    final private Map<TblColRef, DimensionEncoding> encMap = Maps.newHashMap();
+    
+    public CubeDimEncMap(CubeSegment seg) {
+        this.cubeDesc = seg.getCubeDesc();
+        this.seg = seg;
+        this.dictionaryMap = null;
+    }
+    
+    public CubeDimEncMap(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        this.cubeDesc = cubeDesc;
+        this.seg = null;
+        this.dictionaryMap = dictionaryMap;
+    }
+    
+    @Override
+    public DimensionEncoding get(TblColRef col) {
+        DimensionEncoding result = encMap.get(col);
+        if (result == null) {
+            RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
+            if (colDesc.isUsingDictionary()) {
+                // dictionary encoding
+                result = new DictionaryDimEnc(getDictionary(col));
+            }
+            else {
+                // fixed length encoding
+                result = new FixedLenDimEnc(colDesc.getLength());
+            }
+            encMap.put(col, result);
+        }
+        return result;
+    }
+    
+    @Override
+    public Dictionary<String> getDictionary(TblColRef col) {
+        if (seg == null)
+            return dictionaryMap.get(col);
+        else
+            return seg.getDictionary(col);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
index 3510915..987fb55 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
@@ -22,8 +22,6 @@ public class RowConstants {
 
     public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
 
-    // row key fixed length place holder
-    public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
     // row key lower bound
     public static final byte ROWKEY_LOWER_BYTE = 0;
     // row key upper bound

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index 1d57cf9..9af1018 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -18,161 +18,46 @@
 
 package org.apache.kylin.cube.kv;
 
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
+import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Read/Write column values from/into bytes
  *
  * @author yangli9
  */
-@SuppressWarnings("unchecked")
 public class RowKeyColumnIO {
 
-    private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
-
-    private IDictionaryAware IDictionaryAwareness;
+    //private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
 
-    public RowKeyColumnIO(IDictionaryAware IDictionaryAwareness) {
-        this.IDictionaryAwareness = IDictionaryAwareness;
-    }
+    private final IDimensionEncodingMap dimEncMap;
 
-    public IDictionaryAware getIDictionaryAware() {
-        return IDictionaryAwareness;
+    public RowKeyColumnIO(IDimensionEncodingMap dimEncMap) {
+        this.dimEncMap = dimEncMap;
     }
 
     public int getColumnLength(TblColRef col) {
-        return IDictionaryAwareness.getColumnLength(col);
+        return dimEncMap.get(col).getLengthOfEncoding();
     }
 
-    //TODO is type cast really necessary here?
     public Dictionary<String> getDictionary(TblColRef col) {
-        return (Dictionary<String>) IDictionaryAwareness.getDictionary(col);
-    }
-
-    public void writeColumnWithoutDictionary(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength) {
-        if (srcLength >= dstLength) {
-            System.arraycopy(src, srcOffset, dst, dstOffset, dstLength);
-        } else {
-            System.arraycopy(src, srcOffset, dst, dstOffset, srcLength);
-            Arrays.fill(dst, dstOffset + srcLength, dstOffset + dstLength, RowConstants.ROWKEY_PLACE_HOLDER_BYTE);
-        }
+        return dimEncMap.getDictionary(col);
     }
 
-    public void writeColumnWithDictionary(Dictionary<String> dictionary, byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength, int roundingFlag, int defaultValue) {
-        // dict value
-        try {
-            int id = dictionary.getIdFromValueBytes(src, srcOffset, srcLength, roundingFlag);
-            BytesUtil.writeUnsigned(id, dst, dstOffset, dictionary.getSizeOfId());
-        } catch (IllegalArgumentException ex) {
-            Arrays.fill(dst, dstOffset, dstOffset + dstLength, (byte) defaultValue);
-            logger.error("Can't translate value " + Bytes.toString(src, srcOffset, srcLength) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", defaultValue));
-        }
-    }
-
-    public void writeColumn(TblColRef column, byte[] value, int valueLen, byte defaultValue, byte[] output, int outputOffset) {
-        writeColumn(column, value, valueLen, 0, defaultValue, output, outputOffset);
-    }
-
-    public void writeColumn(TblColRef column, byte[] value, int valueLen, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) {
-
-        final Dictionary<String> dict = getDictionary(column);
-        final int columnLen = getColumnLength(column);
+    public void writeColumn(TblColRef col, byte[] value, int valueLen, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) {
+        DimensionEncoding dimEnc = dimEncMap.get(col);
+        if (dimEnc instanceof DictionaryDimEnc)
+            dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue);
 
-        // non-dict value
-        if (dict == null) {
-            byte[] valueBytes = padFixLen(columnLen, value, valueLen);
-            System.arraycopy(valueBytes, 0, output, outputOffset, columnLen);
-            return;
-        }
-
-        // dict value
-        try {
-            int id = dict.getIdFromValueBytes(value, 0, valueLen, roundingFlag);
-            BytesUtil.writeUnsigned(id, output, outputOffset, dict.getSizeOfId());
-        } catch (IllegalArgumentException ex) {
-            for (int i = outputOffset; i < outputOffset + columnLen; i++) {
-                output[i] = defaultValue;
-            }
-            logger.error("Can't translate value " + Bytes.toString(value, 0, valueLen) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", defaultValue));
-        }
-    }
-
-    private byte[] padFixLen(int length, byte[] valueBytes, int valLen) {
-        if (valLen == length) {
-            return valueBytes;
-        } else if (valLen < length) {
-            byte[] newValueBytes = new byte[length];
-            System.arraycopy(valueBytes, 0, newValueBytes, 0, valLen);
-            Arrays.fill(newValueBytes, valLen, length, RowConstants.ROWKEY_PLACE_HOLDER_BYTE);
-            return newValueBytes;
-        } else {
-            return Arrays.copyOf(valueBytes, length);
-        }
+        dimEnc.encode(value, valueLen, output, outputOffset);
     }
 
     public String readColumnString(TblColRef col, byte[] bytes, int offset, int length) {
-        Dictionary<String> dict = getDictionary(col);
-        if (dict == null) {
-            if (isNull(bytes, offset, length)) {
-                return null;
-            }
-            bytes = removeFixLenPad(bytes, offset, length);
-            return Bytes.toString(bytes);
-        } else {
-            int id = BytesUtil.readUnsigned(bytes, offset, length);
-            try {
-                String value = dict.getValueFromId(id);
-                return value;
-            } catch (IllegalArgumentException e) {
-                logger.error("Can't get dictionary value for column " + col.getName() + " (id = " + id + ")");
-                return "";
-            }
-        }
-    }
-
-    public String readColumnString(TblColRef col, byte[] bytes, int bytesLen) {
-        return readColumnString(col, bytes, 0, bytesLen);
-    }
-
-    private boolean isNull(byte[] bytes, int offset, int length) {
-        // all 0xFF is NULL
-        if (length == 0) {
-            return false;
-        }
-        for (int i = 0; i < bytes.length; i++) {
-            if (bytes[i + offset] != AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    private byte[] removeFixLenPad(byte[] bytes, int offset, int length) {
-        int padCount = 0;
-        for (int i = 0; i < length; i++) {
-            if (bytes[i + offset] == RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
-                padCount++;
-            }
-        }
-
-        int size = length - padCount;
-        byte[] stripBytes = new byte[size];
-        int index = 0;
-        for (int i = 0; i < length; i++) {
-            byte vb = bytes[i + offset];
-            if (vb != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
-                stripBytes[index++] = vb;
-            }
-        }
-        return stripBytes;
+        DimensionEncoding dimEnc = dimEncMap.get(col);
+        return dimEnc.decode(bytes, offset, length);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index e4a6a52..d5948d4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -46,7 +46,7 @@ public class RowKeyDecoder {
     public RowKeyDecoder(CubeSegment cubeSegment) {
         this.cubeDesc = cubeSegment.getCubeDesc();
         this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
-        this.colIO = new RowKeyColumnIO(cubeSegment);
+        this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
         this.values = new ArrayList<String>();
     }
 
@@ -77,7 +77,7 @@ public class RowKeyDecoder {
     }
 
     private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {
-        String strValue = colIO.readColumnString(col, valueBytes, length);
+        String strValue = colIO.readColumnString(col, valueBytes, 0, length);
         values.add(strValue);
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index 990cf06..05afdbf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -44,7 +44,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
     public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
         super(cubeSeg, cuboid);
         enableSharding = cubeSeg.isEnableSharding();
-        colIO = new RowKeyColumnIO(cubeSeg);
+        colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap());
         for (TblColRef column : cuboid.getColumns()) {
             bodyLength += colIO.getColumnLength(column);
         }
@@ -160,7 +160,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
             return;
         }
 
-        colIO.writeColumn(column, value, valueLen, this.blankByte, outputValue, outputValueOffset);
+        colIO.writeColumn(column, value, valueLen, 0, this.blankByte, outputValue, outputValueOffset);
     }
 
     protected byte defaultValue() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index 7254df0..3f33505 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -67,17 +67,7 @@ public class RowKeyDesc {
         return getColDesc(col).getBitIndex();
     }
 
-    /**
-     * caller of this method must make sure that col is NOT using dictionary
-     * otherwise this will always return 0 for dict columns
-     * @param col
-     * @return
-     */
-    public int getColumnLength(TblColRef col) {
-        return getColDesc(col).getLength();
-    }
-
-    private RowKeyColDesc getColDesc(TblColRef col) {
+    public RowKeyColDesc getColDesc(TblColRef col) {
         RowKeyColDesc desc = columnMap.get(col);
         if (desc == null)
             throw new NullPointerException("Column " + col + " does not exist in row key desc");

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index bcb2caf..7392e4c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -43,7 +43,6 @@ import javax.annotation.Nullable;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
@@ -54,6 +53,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.source.ReadableTable;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
index c0fe939..57e0d8b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
@@ -19,18 +19,12 @@
 package org.apache.kylin.gridtable;
 
 import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DimensionEncoding;
 
 public class DefaultGTComparator implements IGTComparator {
     @Override
     public boolean isNull(ByteArray code) {
-        // all 0xff is null
-        byte[] array = code.array();
-        for (int i = 0, j = code.offset(), n = code.length(); i < n; i++, j++) {
-            if (array[j] != Dictionary.NULL)
-                return false;
-        }
-        return true;
+        return DimensionEncoding.isNull(code.array(), code.offset(), code.length());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index e4dd8f5..4a9c2d3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -23,12 +23,12 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.HashSet;
 
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.After;
 import org.junit.Before;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
index c25bad7..8d8366e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
@@ -26,10 +26,10 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.AfterClass;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
index 832584c..80e3df1 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
@@ -33,10 +33,10 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.AfterClass;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index d96de4f..88573c6 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -33,7 +33,6 @@ import java.util.concurrent.Future;
 import org.apache.commons.io.FileUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -42,6 +41,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
 import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 674aa15..517299f 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -17,7 +17,7 @@
 
 package org.apache.kylin.gridtable;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -25,17 +25,18 @@ import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.BitSet;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.gridtable.CubeCodeSystem;
 import org.apache.kylin.dict.NumberDictionaryBuilder;
 import org.apache.kylin.dict.StringBytesConverter;
 import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.GTInfo.Builder;
 import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -55,7 +56,6 @@ import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 
 public class DictGridTableTest {
 
@@ -466,12 +466,12 @@ public class DictGridTableTest {
         return info;
     }
 
-    @SuppressWarnings("rawtypes")
+    @SuppressWarnings("unchecked")
     private static CubeCodeSystem newDictCodeSystem() {
-        Map<Integer, Dictionary> dictionaryMap = Maps.newHashMap();
-        dictionaryMap.put(1, newDictionaryOfInteger());
-        dictionaryMap.put(2, newDictionaryOfString());
-        return new CubeCodeSystem(dictionaryMap);
+        DimensionEncoding[] dimEncs = new DimensionEncoding[3];
+        dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
+        dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString());
+        return new CubeCodeSystem(dimEncs);
     }
 
     @SuppressWarnings("rawtypes")

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 62b06aa..b666229 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -30,7 +30,7 @@ import java.io.UnsupportedEncodingException;
 import java.util.Date;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 
 /**
  * A dictionary for date string (date only, no time).

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
index 158ce86..06758c0 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
@@ -21,7 +21,7 @@ package org.apache.kylin.dict;
 import java.nio.ByteBuffer;
 
 import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.metadata.filter.IFilterCodeSystem;
 
 /**
@@ -44,7 +44,7 @@ public class DictCodeSystem implements IFilterCodeSystem<String> {
 
         String v = value;
         for (int i = 0, n = v.length(); i < n; i++) {
-            if ((byte) v.charAt(i) != Dictionary.NULL)
+            if ((byte) v.charAt(i) != DimensionEncoding.NULL)
                 return false;
         }
         return true;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 4b01e60..df6781a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -27,8 +27,8 @@ import java.util.List;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.source.ReadableTable;
 import org.slf4j.Logger;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 4fba59a..f38c95f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict;
 
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.source.ReadableTable.TableSignature;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
index 69b29fe..47844be 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
@@ -24,8 +24,8 @@ import java.io.IOException;
 
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.dimension.Dictionary;
 
 /**
  * @author yangli9

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index d49e43d..ce04b55 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.DataModelDesc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
index 72dc995..6d51f4a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -28,7 +28,7 @@ import java.io.OutputStream;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 
 /**
  */

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
deleted file mode 100644
index 4586163..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- * Class that implement this interface has the ability to help dictionary encoding and decoding
- */
-public interface IDictionaryAware {
-
-    public abstract int getColumnLength(TblColRef col);
-
-    public abstract Dictionary<?> getDictionary(TblColRef col);
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index df7b1c6..96448a4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -21,7 +21,7 @@ package org.apache.kylin.dict;
 import com.google.common.collect.Lists;
 
 import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 
 import java.io.IOException;
 import java.util.List;

http://git-wip-us.apache.org/repos/asf/kylin/blob/75027ed6/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index 3c96d08..ab38b13 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -25,7 +25,7 @@ import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
 
 import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.Dictionary;
 
 /**
  */


Mime
View raw message