kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [47/50] incubator-kylin git commit: KYLIN-625, filter convert pass
Date Sat, 28 Mar 2015 00:05:24 GMT
KYLIN-625, filter convert pass


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d7fc2312
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d7fc2312
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d7fc2312

Branch: refs/heads/streaming-localdict
Commit: d7fc2312c4c800cb2fec5264b6997feda6527521
Parents: 24acccc
Author: Li, Yang <yangli9@ebay.com>
Authored: Fri Mar 27 18:26:39 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Fri Mar 27 18:26:39 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/util/ByteArray.java |   9 +-
 .../metadata/filter/ColumnTupleFilter.java      |   6 +-
 .../metadata/filter/CompareTupleFilter.java     |   2 +-
 .../metadata/filter/ConstantTupleFilter.java    |   4 +-
 .../metadata/filter/LogicalTupleFilter.java     |   2 +-
 .../metadata/serializer/DataTypeSerializer.java |   3 +-
 .../gridtable/GTDictionaryCodeSystem.java       |  47 +++-
 .../kylin/storage/gridtable/GTRecord.java       |  29 ++-
 .../storage/gridtable/GTSampleCodeSystem.java   |   5 +-
 .../kylin/storage/gridtable/GTScanRange.java    |  11 +-
 .../kylin/storage/gridtable/GTScanRequest.java  |   8 +-
 .../apache/kylin/storage/gridtable/GTUtil.java  |  35 ++-
 .../storage/gridtable/DictGridTableTest.java    | 214 +++++++++++++++++++
 .../storage/gridtable/GTInvertedIndexTest.java  | 165 --------------
 .../kylin/storage/gridtable/GridTableTest.java  | 208 ------------------
 .../storage/gridtable/SimpleGridTableTest.java  | 208 ++++++++++++++++++
 .../gridtable/SimpleInvertedIndexTest.java      | 165 ++++++++++++++
 17 files changed, 690 insertions(+), 431 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index d09b350..8c6ae91 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -30,7 +30,7 @@ public class ByteArray implements Comparable<ByteArray> {
     public static ByteArray allocate(int length) {
         return new ByteArray(new byte[length]);
     }
-    
+
     public static ByteArray copyOf(byte[] array, int offset, int length) {
         byte[] space = new byte[length];
         System.arraycopy(array, offset, space, 0, length);
@@ -52,7 +52,7 @@ public class ByteArray implements Comparable<ByteArray> {
     }
 
     public ByteArray(byte[] data) {
-        set(data, 0, data.length);
+        set(data, 0, data == null ? 0 : data.length);
     }
 
     public ByteArray(byte[] data, int offset, int length) {
@@ -148,7 +148,10 @@ public class ByteArray implements Comparable<ByteArray> {
 
     @Override
     public String toString() {
-        return Bytes.toString(data, offset, length);
+        if (data == null)
+            return null;
+        else
+            return Bytes.toStringBinary(data, offset, length);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index f689ccb..fde41b1 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -63,7 +63,7 @@ public class ColumnTupleFilter extends TupleFilter {
 
     @Override
     public String toString() {
-        return "ColumnFilter [column=" + columnRef + "]";
+        return "" + columnRef;
     }
 
     @Override
@@ -79,7 +79,7 @@ public class ColumnTupleFilter extends TupleFilter {
 
     @Override
     public Collection<?> getValues() {
-        this.values.set(0, (String) this.tupleValue);
+        this.values.set(0, this.tupleValue);
         return this.values;
     }
 
@@ -114,7 +114,7 @@ public class ColumnTupleFilter extends TupleFilter {
             table = new TableDesc();
             table.setName(tableName);
         }
-        
+
         column.setId(BytesUtil.readUTFString(buffer));
         column.setName(BytesUtil.readUTFString(buffer));
         column.setDatatype(BytesUtil.readUTFString(buffer));

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 2b68469..57b50b7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -126,7 +126,7 @@ public class CompareTupleFilter extends TupleFilter {
 
     @Override
     public String toString() {
-        return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
+        return column + " " + operator + " " + conditionValues;
     }
 
     // TODO requires generalize, currently only evaluates COLUMN {op} CONST

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index f372b4a..cc3add2 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -34,7 +34,7 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
 public class ConstantTupleFilter extends TupleFilter {
 
     public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
-    public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
+    public static final ConstantTupleFilter TRUE = new ConstantTupleFilter((Object) null); // not sure of underlying code system, null is the only value that applies to all types
 
     private Collection<Object> constantValues;
 
@@ -60,7 +60,7 @@ public class ConstantTupleFilter extends TupleFilter {
 
     @Override
     public String toString() {
-        return "ConstantFilter [constant=" + constantValues + "]";
+        return "" + constantValues;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1844392..4d38565 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -67,7 +67,7 @@ public class LogicalTupleFilter extends TupleFilter {
 
     @Override
     public String toString() {
-        return "LogicalFilter [operator=" + operator + ", children=" + children + "]";
+        return operator + " " + children;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
index 63d4ddd..aafb1c2 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
@@ -70,9 +70,10 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
     /** peek into buffer and return the length of serialization */
     abstract public int peekLength(ByteBuffer in);
     
-    /** convert from String to obj */
+    /** convert from String to obj (string often come as byte[] in mapred) */
     abstract public T valueOf(byte[] value);
     
+    /** convert from String to obj */
     public T valueOf(String value) {
         try {
             return valueOf(value.getBytes("UTF-8"));

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
index ada4ed7..c94c604 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTDictionaryCodeSystem.java
@@ -2,10 +2,13 @@ package org.apache.kylin.storage.gridtable;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.metadata.filter.IFilterCodeSystem;
 import org.apache.kylin.metadata.measure.MeasureAggregator;
 import org.apache.kylin.metadata.serializer.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.nio.ByteBuffer;
 import java.util.Map;
@@ -15,13 +18,15 @@ import java.util.Map;
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class GTDictionaryCodeSystem implements IGTCodeSystem {
+    private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
+
     private GTInfo info;
-    private Map<Integer, Dictionary> dictionaryMaps = null; // key: column index; value: dictionary for this column;
+    private Map<Integer, Dictionary> dictionaryMap = null; // key: column index; value: dictionary for this column;
     private IFilterCodeSystem<ByteArray> filterCS;
     private DataTypeSerializer[] serializers;
 
-    public GTDictionaryCodeSystem(Map<Integer, Dictionary> dictionaryMaps) {
-        this.dictionaryMaps = dictionaryMaps;
+    public GTDictionaryCodeSystem(Map<Integer, Dictionary> dictionaryMap) {
+        this.dictionaryMap = dictionaryMap;
     }
 
     @Override
@@ -30,8 +35,8 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
 
         serializers = new DataTypeSerializer[info.nColumns];
         for (int i = 0; i < info.nColumns; i++) {
-            if (dictionaryMaps.get(i) != null) {
-                serializers[i] = new DictionarySerializer(dictionaryMaps.get(i));
+            if (dictionaryMap.get(i) != null) {
+                serializers[i] = new DictionarySerializer(dictionaryMap.get(i));
             } else {
                 serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
             }
@@ -56,7 +61,10 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
 
             @Override
             public void serialize(ByteArray code, ByteBuffer buffer) {
-                BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+                if (code == null)
+                    BytesUtil.writeByteArray(null, 0, 0, buffer);
+                else
+                    BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
             }
 
             @Override
@@ -78,16 +86,33 @@ public class GTDictionaryCodeSystem implements IGTCodeSystem {
 
     @Override
     public void encodeColumnValue(int col, Object value, ByteBuffer buf) {
-        serializers[col].serialize(value, buf);
+        encodeColumnValue(col, value, 0, buf);
     }
 
     @Override
     public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
+        // this is a bit too complicated, but encoding only happens once at build time, so it is OK
         DataTypeSerializer serializer = serializers[col];
-        if (serializer instanceof DictionarySerializer) {
-            ((DictionarySerializer) serializer).serializeWithRounding(value,  roundingFlag, buf);
-        } else {
-            serializer.serialize(value,  buf);
+        try {
+            if (serializer instanceof DictionarySerializer) {
+                ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
+            } else {
+                serializer.serialize(value, buf);
+            }
+        } catch (ClassCastException ex) {
+            // try convert string into a correct object type
+            try {
+                if (value instanceof String) {
+                    Object converted = serializer.valueOf((String) value);
+                    if ((converted instanceof String) == false) {
+                        encodeColumnValue(col, converted, roundingFlag, buf);
+                        return;
+                    }
+                }
+            } catch (Throwable e) {
+                logger.error("Fail to encode value '" + value + "'", e);
+            }
+            throw ex;
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 605a469..aeefc2b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -21,7 +21,7 @@ public class GTRecord implements Comparable<GTRecord> {
             this.cols[i] = new ByteArray();
         this.maskForEqualHashComp = info.colAll;
     }
-    
+
     public ByteArray get(int i) {
         return cols[i];
     }
@@ -51,16 +51,17 @@ public class GTRecord implements Comparable<GTRecord> {
 
     /** decode and return the values of this record */
     public Object[] getValues() {
-        return getValues(new Object[info.nColumns]);
+        return getValues(info.colAll, new Object[info.nColumns]);
     }
 
     /** decode and return the values of this record */
-    public Object[] getValues(Object[] result) {
-        for (int i = 0; i < info.nColumns; i++) {
-            if (cols[i].array() == null)
+    public Object[] getValues(BitSet selectedColumns, Object[] result) {
+        assert selectedColumns.cardinality() <= result.length;
+        for (int i = 0, c = selectedColumns.nextSetBit(0); c >= 0; i++, c = selectedColumns.nextSetBit(c + 1)) {
+            if (cols[c].array() == null)
                 result[i] = null;
             else
-                result[i] = info.codeSystem.decodeColumnValue(i, cols[i].asBuffer());
+                result[i] = info.codeSystem.decodeColumnValue(c, cols[c].asBuffer());
         }
         return result;
     }
@@ -92,11 +93,11 @@ public class GTRecord implements Comparable<GTRecord> {
     public BitSet maskForEqualHashComp() {
         return maskForEqualHashComp;
     }
-    
+
     public void maskForEqualHashComp(BitSet set) {
         this.maskForEqualHashComp = set;
     }
-    
+
     @Override
     public boolean equals(Object obj) {
         if (this == obj)
@@ -132,7 +133,7 @@ public class GTRecord implements Comparable<GTRecord> {
         assert this.info == o.info;
         assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance
         IFilterCodeSystem<ByteArray> cs = info.codeSystem.getFilterCodeSystem();
-        
+
         int comp = 0;
         for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
             comp = cs.compare(cols[i], o.cols[i]);
@@ -141,10 +142,16 @@ public class GTRecord implements Comparable<GTRecord> {
         }
         return comp;
     }
-    
+
     @Override
     public String toString() {
-        return Arrays.toString(getValues());
+        return toString(info.colAll);
+    }
+    
+    public String toString(BitSet selectedColumns) {
+        Object[] values = new Object[selectedColumns.cardinality()];
+        getValues(selectedColumns, values);
+        return Arrays.toString(values);
     }
 
     // ============================================================================

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index 083d8c2..aea4e49 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -54,7 +54,10 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
 
             @Override
             public void serialize(ByteArray code, ByteBuffer buffer) {
-                BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+                if (code == null)
+                    BytesUtil.writeByteArray(null, 0, 0, buffer);
+                else
+                    BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
             }
 
             @Override

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
index 08513f7..b09a01d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
@@ -14,12 +14,9 @@ public class GTScanRange {
     }
 
     public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> hbaseFuzzyKeys) {
-        assert pkStart.info == pkEnd.info;
-        assert pkStart.maskForEqualHashComp() == pkStart.info.primaryKey;
-        assert pkEnd.maskForEqualHashComp() == pkEnd.info.primaryKey;
         this.pkStart = pkStart;
         this.pkEnd = pkEnd;
-        this.hbaseFuzzyKeys = hbaseFuzzyKeys == null ? Collections.<GTRecord>emptyList() : hbaseFuzzyKeys;
+        this.hbaseFuzzyKeys = hbaseFuzzyKeys == null ? Collections.<GTRecord> emptyList() : hbaseFuzzyKeys;
     }
 
     @Override
@@ -58,4 +55,10 @@ public class GTScanRange {
             return false;
         return true;
     }
+
+    @Override
+    public String toString() {
+        return (pkStart == null ? "null" : pkStart.toString(pkStart.info.primaryKey)) //
+                + "-" + (pkEnd == null ? "null" : pkEnd.toString(pkEnd.info.primaryKey));
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
index c92cba4..b71032c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
@@ -1,5 +1,6 @@
 package org.apache.kylin.storage.gridtable;
 
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Set;
 
@@ -87,7 +88,7 @@ public class GTScanRequest {
         // un-evaluatable filter must be removed
         if (TupleFilter.isEvaluableRecursively(filterPushDown) == false) {
             Set<TblColRef> unevaluableColumns = Sets.newHashSet();
-            filterPushDown = GTUtil.convertFilterUnevaluatable(filterPushDown, unevaluableColumns);
+            filterPushDown = GTUtil.convertFilterUnevaluatable(filterPushDown, info, unevaluableColumns);
 
             // columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation
             if (hasAggregation()) {
@@ -138,4 +139,9 @@ public class GTScanRequest {
         return aggrMetricsFuncs;
     }
 
+    @Override
+    public String toString() {
+        return "GTScanRequest [range=" + range + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index 1fb0376..7d042eb 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -34,18 +34,18 @@ public class GTUtil {
         return new TblColRef(desc);
     }
 
-    public static TupleFilter convertFilterUnevaluatable(TupleFilter rootFilter, //
-            final Set<TblColRef> unevaluatableColumnCollector) {
-        return convertFilter(rootFilter, null, null, false, unevaluatableColumnCollector);
+    public static TupleFilter convertFilterUnevaluatable(TupleFilter rootFilter, GTInfo info, //
+            Set<TblColRef> unevaluatableColumnCollector) {
+        return convertFilter(rootFilter, info, null, false, unevaluatableColumnCollector);
     }
 
-    public static TupleFilter convertFilterConstants(TupleFilter rootFilter, final GTInfo info) {
+    public static TupleFilter convertFilterConstants(TupleFilter rootFilter, GTInfo info) {
         return convertFilter(rootFilter, info, null, true, null);
     }
 
-    public static TupleFilter convertFilterColumnsAndConstants(TupleFilter rootFilter, final GTInfo info, //
-            final Map<TblColRef, Integer> colMapping, //
-            final Set<TblColRef> unevaluatableColumnCollector) {
+    public static TupleFilter convertFilterColumnsAndConstants(TupleFilter rootFilter, GTInfo info, //
+            Map<TblColRef, Integer> colMapping, //
+            Set<TblColRef> unevaluatableColumnCollector) {
         return convertFilter(rootFilter, info, colMapping, true, unevaluatableColumnCollector);
     }
 
@@ -68,6 +68,12 @@ public class GTUtil {
                     return ConstantTupleFilter.TRUE;
                 }
 
+                // shortcut for unEvaluatable filter
+                if (filter.isEvaluable() == false) {
+                    TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
+                    return ConstantTupleFilter.TRUE;
+                }
+
                 // map to column onto grid table
                 if (colMapping != null && filter instanceof ColumnTupleFilter) {
                     ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
@@ -75,18 +81,9 @@ public class GTUtil {
                     return new ColumnTupleFilter(info.colRef(gtColIdx));
                 }
 
-                // below consider compare filter only
-                if (filter instanceof CompareTupleFilter) {
-
-                    // shortcut for unEvaluatable compare filter
-                    if (TupleFilter.isEvaluableRecursively(filter) == false) {
-                        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
-                        return ConstantTupleFilter.TRUE;
-                    }
-
-                    if (encodeConstants) {
-                        return encodeConstants((CompareTupleFilter) filter);
-                    }
+                // encode constants
+                if (encodeConstants && filter instanceof CompareTupleFilter) {
+                    return encodeConstants((CompareTupleFilter) filter);
                 }
 
                 return filter;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
new file mode 100644
index 0000000..46ec66c
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
@@ -0,0 +1,214 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Map;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTInfo.Builder;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class DictGridTableTest {
+
+    @Test
+    public void test() throws IOException {
+        GridTable table = newTestTable();
+        verifyFirstRow(table);
+        verifyScanWithUnevaluatableFilter(table);
+        verifyScanWithEvaluatableFilter(table);
+    }
+
+    private void verifyFirstRow(GridTable table) throws IOException {
+        doScanAndVerify(table, new GTScanRequest(table.getInfo()), "[1421193600000, 30, Yang, 10, 10.5]");
+    }
+
+    private void verifyScanWithUnevaluatableFilter(GridTable table) throws IOException {
+        GTInfo info = table.getInfo();
+
+        CompareTupleFilter fcomp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+        ExtractTupleFilter funevaluatable = unevaluatable(info.colRef(1));
+        LogicalTupleFilter filter = and(fcomp, funevaluatable);
+
+        GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+        // note the unEvaluatable column 1 in filter is added to group by
+        assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+        
+        doScanAndVerify(table, req, "[1421280000000, 20, null, 20, null]");
+    }
+    
+    private void verifyScanWithEvaluatableFilter(GridTable table) throws IOException {
+        GTInfo info = table.getInfo();
+
+        CompareTupleFilter fcomp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+        CompareTupleFilter fcomp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+        LogicalTupleFilter filter = and(fcomp1, fcomp2);
+
+        GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+        // note the evaluatable column 1 in filter is added to returned columns but not in group by
+        assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.2 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+        
+        doScanAndVerify(table, req, "[1421280000000, 30, null, 30, null]", "[1421366400000, 20, null, 40, null]");
+    }
+
+    private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
+        System.out.println(req);
+        IGTScanner scanner = table.scan(req);
+        int i = 0;
+        for (GTRecord r : scanner) {
+            System.out.println(r);
+            if (verifyRows != null && i < verifyRows.length) {
+                assertEquals(verifyRows[i], r.toString());
+            }
+            i++;
+        }
+        scanner.close();
+    }
+
+    private Object enc(GTInfo info, int col, String value) {
+        ByteBuffer buf = ByteBuffer.allocate(info.maxRecordLength);
+        info.codeSystem.encodeColumnValue(col, value, buf);
+        return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
+    }
+
+    private ExtractTupleFilter unevaluatable(TblColRef col) {
+        ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
+        r.addChild(new ColumnTupleFilter(col));
+        return r;
+    }
+
+    private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object value) {
+        CompareTupleFilter result = new CompareTupleFilter(op);
+        result.addChild(new ColumnTupleFilter(col));
+        result.addChild(new ConstantTupleFilter(value));
+        return result;
+    }
+
+    private LogicalTupleFilter and(TupleFilter... children) {
+        return logic(FilterOperatorEnum.AND, children);
+    }
+
+    private LogicalTupleFilter or(TupleFilter... children) {
+        return logic(FilterOperatorEnum.AND, children);
+    }
+
+    private LogicalTupleFilter not(TupleFilter child) {
+        return logic(FilterOperatorEnum.AND, child);
+    }
+
+    private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+        LogicalTupleFilter result = new LogicalTupleFilter(op);
+        for (TupleFilter c : children) {
+            result.addChild(c);
+        }
+        return result;
+    }
+
+    static GridTable newTestTable() throws IOException {
+        GTInfo info = newInfo();
+        GTSimpleMemStore store = new GTSimpleMemStore(info);
+        GridTable table = new GridTable(info, store);
+
+        GTRecord r = new GTRecord(table.getInfo());
+        GTBuilder builder = table.rebuild();
+
+        builder.write(r.setValues("2015-01-14", "30", "Yang", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-14", "30", "Luke", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "30", "Xu", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "20", "Dong", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "20", "Jason", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "30", "George", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+
+        return table;
+    }
+
+    static GTInfo newInfo() {
+        Builder builder = GTInfo.builder();
+        builder.setCodeSystem(newDictCodeSystem());
+        builder.setColumns( //
+                DataType.getInstance("timestamp"), //
+                DataType.getInstance("integer"), //
+                DataType.getInstance("varchar"), //
+                DataType.getInstance("bigint"), //
+                DataType.getInstance("decimal") //
+        );
+        builder.setPrimaryKey(setOf(0));
+        builder.setColumnPreferIndex(setOf(0));
+        builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
+        builder.enableRowBlock(4);
+        GTInfo info = builder.build();
+        return info;
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static GTDictionaryCodeSystem newDictCodeSystem() {
+        Map<Integer, Dictionary> dictionaryMap = Maps.newHashMap();
+        dictionaryMap.put(1, newDictionaryOfInteger());
+        dictionaryMap.put(2, newDictionaryOfString());
+        return new GTDictionaryCodeSystem(dictionaryMap);
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static Dictionary newDictionaryOfString() {
+        TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+        builder.addValue("Dong");
+        builder.addValue("George");
+        builder.addValue("Jason");
+        builder.addValue("Kejia");
+        builder.addValue("Luke");
+        builder.addValue("Mahone");
+        builder.addValue("Qianhao");
+        builder.addValue("Shaofeng");
+        builder.addValue("Xu");
+        builder.addValue("Yang");
+        return builder.build(0);
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static Dictionary newDictionaryOfInteger() {
+        NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+        builder.addValue("10");
+        builder.addValue("20");
+        builder.addValue("30");
+        builder.addValue("40");
+        builder.addValue("50");
+        builder.addValue("60");
+        builder.addValue("70");
+        builder.addValue("80");
+        builder.addValue("90");
+        builder.addValue("100");
+        return builder.build(0);
+    }
+
+    private static BitSet setOf(int... values) {
+        BitSet set = new BitSet();
+        for (int i : values)
+            set.set(i);
+        return set;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
deleted file mode 100644
index 1460039..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
+++ /dev/null
@@ -1,165 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import static org.junit.Assert.*;
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.serializer.StringSerializer;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class GTInvertedIndexTest {
-
-    GTInfo info;
-    GTInvertedIndex index;
-    ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
-    ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
-    public GTInvertedIndexTest() {
-        
-        info = GridTableTest.advancedInfo();
-        TblColRef colA = info.colRef(0);
-        
-        // block i contains value "i", the last is NULL
-        index = new GTInvertedIndex(info);
-        GTRowBlock mockBlock = GTRowBlock.allocate(info);
-        GTRowBlock.Writer writer = mockBlock.getWriter();
-        GTRecord record = new GTRecord(info);
-        for (int i = 0; i < 10; i++) {
-            record.setValues(i < 9 ? "" + i : null, "", "", new LongWritable(0), new BigDecimal(0));
-            for (int j = 0; j < info.getRowBlockSize(); j++) {
-                writer.append(record);
-            }
-            writer.readyForFlush();
-            index.add(mockBlock);
-            
-            writer.clearForNext();
-        }
-        
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
-        basicResults.add(set(9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
-        basicResults.add(set(0));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
-        basicResults.add(set(0, 5));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
-        basicResults.add(set(0, 1, 2));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
-        basicResults.add(set(0, 1, 2, 3));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
-        basicResults.add(set(4, 5, 6, 7, 8));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
-        basicResults.add(set(3, 4, 5, 6, 7, 8));
-    }
-
-    @Test
-    public void testBasics() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
-        }
-    }
-
-    @Test
-    public void testLogicalAnd() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.retainAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testLogicalOr() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.addAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testNotEvaluable() {
-        ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-        
-        CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
-        assertEquals(all, index.filter(notEvaluable));
-
-        LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
-        assertEquals(all, index.filter(or));
-
-        LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
-        assertEquals(basicResults.get(0), index.filter(and));
-    }
-
-    public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
-        CompareTupleFilter filter = new CompareTupleFilter(op);
-        filter.addChild(columnFilter(col));
-        for (int i : ids) {
-            filter.addChild(constFilter(i));
-        }
-        return filter;
-    }
-
-    public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
-        LogicalTupleFilter filter = new LogicalTupleFilter(op);
-        for (TupleFilter f : filters)
-            filter.addChild(f);
-        return filter;
-    }
-
-    public static ColumnTupleFilter columnFilter(TblColRef col) {
-        return new ColumnTupleFilter(col);
-    }
-
-    public static ConstantTupleFilter constFilter(int id) {
-        byte[] space = new byte[10];
-        ByteBuffer buf = ByteBuffer.wrap(space);
-        StringSerializer stringSerializer = new StringSerializer();
-        stringSerializer.serialize("" + id, buf);
-        ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
-        return new ConstantTupleFilter(data);
-    }
-
-    public static ConciseSet set(int... ints) {
-        ConciseSet set = new ConciseSet();
-        for (int i : ints)
-            set.add(i);
-        return set;
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
deleted file mode 100644
index 6561c6e..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.BitSet;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.storage.gridtable.GTInfo.Builder;
-import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
-import org.junit.Test;
-
-public class GridTableTest {
-
-    @Test
-    public void testBasics() throws IOException {
-        GTInfo info = basicInfo();
-        GTSimpleMemStore store = new GTSimpleMemStore(info);
-        GridTable table = new GridTable(info, store);
-
-        GTBuilder builder = rebuild(table);
-        IGTScanner scanner = scan(table);
-        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
-        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
-    }
-
-    @Test
-    public void testAdvanced() throws IOException {
-        GTInfo info = advancedInfo();
-        GTSimpleMemStore store = new GTSimpleMemStore(info);
-        GridTable table = new GridTable(info, store);
-
-        GTBuilder builder = rebuild(table);
-        IGTScanner scanner = scan(table);
-        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
-        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
-    }
-
-    @Test
-    public void testAggregate() throws IOException {
-        GTInfo info = advancedInfo();
-        GTSimpleMemStore store = new GTSimpleMemStore(info);
-        GridTable table = new GridTable(info, store);
-
-        GTBuilder builder = rebuild(table);
-        IGTScanner scanner = scanAndAggregate(table);
-        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
-        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
-    }
-
-    @Test
-    public void testAppend() throws IOException {
-        GTInfo info = advancedInfo();
-        GTSimpleMemStore store = new GTSimpleMemStore(info);
-        GridTable table = new GridTable(info, store);
-
-        rebuildViaAppend(table);
-        IGTScanner scanner = scan(table);
-        assertEquals(3, scanner.getScannedRowBlockCount());
-        assertEquals(10, scanner.getScannedRowCount());
-    }
-
-    private IGTScanner scan(GridTable table) throws IOException {
-        GTScanRequest req = new GTScanRequest(table.getInfo());
-        IGTScanner scanner = table.scan(req);
-        for (GTRecord r : scanner) {
-            Object[] v = r.getValues();
-            assertTrue(((String) v[0]).startsWith("2015-"));
-            assertTrue(((String) v[2]).equals("Food"));
-            assertTrue(((LongWritable) v[3]).get() == 10);
-            assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
-            System.out.println(r);
-        }
-        scanner.close();
-        System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
-        System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
-        return scanner;
-    }
-
-    private IGTScanner scanAndAggregate(GridTable table) throws IOException {
-        GTScanRequest req = new GTScanRequest(table.getInfo(), null, setOf(0, 2), setOf(3, 4), new String[] { "count", "sum" }, null);
-        IGTScanner scanner = table.scan(req);
-        int i = 0;
-        for (GTRecord r : scanner) {
-            Object[] v = r.getValues();
-            switch (i) {
-            case 0:
-                assertTrue(((LongWritable) v[3]).get() == 20);
-                assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
-                break;
-            case 1:
-                assertTrue(((LongWritable) v[3]).get() == 30);
-                assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
-                break;
-            case 2:
-                assertTrue(((LongWritable) v[3]).get() == 40);
-                assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
-                break;
-            case 3:
-                assertTrue(((LongWritable) v[3]).get() == 10);
-                assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
-                break;
-            default:
-                fail();
-            }
-            i++;
-            System.out.println(r);
-        }
-        scanner.close();
-        System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
-        System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
-        return scanner;
-    }
-
-    static GTBuilder rebuild(GridTable table) throws IOException {
-        GTRecord r = new GTRecord(table.getInfo());
-        GTBuilder builder = table.rebuild();
-
-        builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.close();
-
-        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
-        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-        return builder;
-    }
-
-    static void rebuildViaAppend(GridTable table) throws IOException {
-        GTRecord r = new GTRecord(table.getInfo());
-        GTBuilder builder;
-
-        builder = table.append();
-        builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.close();
-        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
-        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
-        builder = table.append();
-        builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.close();
-        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
-        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
-        builder = table.append();
-        builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.close();
-        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
-        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-
-        builder = table.append();
-        builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
-        builder.close();
-        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
-        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
-    }
-
-    static GTInfo basicInfo() {
-        Builder builder = infoBuilder();
-        GTInfo info = builder.build();
-        return info;
-    }
-
-    static GTInfo advancedInfo() {
-        Builder builder = infoBuilder();
-        builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
-        builder.enableRowBlock(4);
-        GTInfo info = builder.build();
-        return info;
-    }
-
-    private static Builder infoBuilder() {
-        Builder builder = GTInfo.builder();
-        builder.setCodeSystem(new GTSampleCodeSystem());
-        builder.setColumns( //
-                DataType.getInstance("varchar"), //
-                DataType.getInstance("varchar"), //
-                DataType.getInstance("varchar"), //
-                DataType.getInstance("bigint"), //
-                DataType.getInstance("decimal") //
-        );
-        builder.setPrimaryKey(setOf(0));
-        builder.setColumnPreferIndex(setOf(0));
-        return builder;
-    }
-
-    private static BitSet setOf(int... values) {
-        BitSet set = new BitSet();
-        for (int i : values)
-            set.set(i);
-        return set;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
new file mode 100644
index 0000000..c5878b3
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
@@ -0,0 +1,208 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.BitSet;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.storage.gridtable.GTInfo.Builder;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.junit.Test;
+
+public class SimpleGridTableTest {
+
+    @Test
+    public void testBasics() throws IOException {
+        GTInfo info = basicInfo();
+        GTSimpleMemStore store = new GTSimpleMemStore(info);
+        GridTable table = new GridTable(info, store);
+
+        GTBuilder builder = rebuild(table);
+        IGTScanner scanner = scan(table);
+        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+    }
+
+    @Test
+    public void testAdvanced() throws IOException {
+        GTInfo info = advancedInfo();
+        GTSimpleMemStore store = new GTSimpleMemStore(info);
+        GridTable table = new GridTable(info, store);
+
+        GTBuilder builder = rebuild(table);
+        IGTScanner scanner = scan(table);
+        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+    }
+
+    @Test
+    public void testAggregate() throws IOException {
+        GTInfo info = advancedInfo();
+        GTSimpleMemStore store = new GTSimpleMemStore(info);
+        GridTable table = new GridTable(info, store);
+
+        GTBuilder builder = rebuild(table);
+        IGTScanner scanner = scanAndAggregate(table);
+        assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount());
+        assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
+    }
+
+    @Test
+    public void testAppend() throws IOException {
+        GTInfo info = advancedInfo();
+        GTSimpleMemStore store = new GTSimpleMemStore(info);
+        GridTable table = new GridTable(info, store);
+
+        rebuildViaAppend(table);
+        IGTScanner scanner = scan(table);
+        assertEquals(3, scanner.getScannedRowBlockCount());
+        assertEquals(10, scanner.getScannedRowCount());
+    }
+
+    private IGTScanner scan(GridTable table) throws IOException {
+        GTScanRequest req = new GTScanRequest(table.getInfo());
+        IGTScanner scanner = table.scan(req);
+        for (GTRecord r : scanner) {
+            Object[] v = r.getValues();
+            assertTrue(((String) v[0]).startsWith("2015-"));
+            assertTrue(((String) v[2]).equals("Food"));
+            assertTrue(((LongWritable) v[3]).get() == 10);
+            assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
+            System.out.println(r);
+        }
+        scanner.close();
+        System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
+        System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
+        return scanner;
+    }
+
+    private IGTScanner scanAndAggregate(GridTable table) throws IOException {
+        GTScanRequest req = new GTScanRequest(table.getInfo(), null, setOf(0, 2), setOf(3, 4), new String[] { "count", "sum" }, null);
+        IGTScanner scanner = table.scan(req);
+        int i = 0;
+        for (GTRecord r : scanner) {
+            Object[] v = r.getValues();
+            switch (i) {
+            case 0:
+                assertTrue(((LongWritable) v[3]).get() == 20);
+                assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
+                break;
+            case 1:
+                assertTrue(((LongWritable) v[3]).get() == 30);
+                assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
+                break;
+            case 2:
+                assertTrue(((LongWritable) v[3]).get() == 40);
+                assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
+                break;
+            case 3:
+                assertTrue(((LongWritable) v[3]).get() == 10);
+                assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
+                break;
+            default:
+                fail();
+            }
+            i++;
+            System.out.println(r);
+        }
+        scanner.close();
+        System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount());
+        System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
+        return scanner;
+    }
+
+    static GTBuilder rebuild(GridTable table) throws IOException {
+        GTRecord r = new GTRecord(table.getInfo());
+        GTBuilder builder = table.rebuild();
+
+        builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+
+        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+        return builder;
+    }
+
+    static void rebuildViaAppend(GridTable table) throws IOException {
+        GTRecord r = new GTRecord(table.getInfo());
+        GTBuilder builder;
+
+        builder = table.append();
+        builder.write(r.setValues("2015-01-14", "Yang", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-14", "Luke", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "Xu", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-15", "Dong", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+        builder = table.append();
+        builder.write(r.setValues("2015-01-15", "Jason", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "Mahone", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "Shaofeng", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+        builder = table.append();
+        builder.write(r.setValues("2015-01-16", "Qianhao", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.write(r.setValues("2015-01-16", "George", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+
+        builder = table.append();
+        builder.write(r.setValues("2015-01-17", "Kejia", "Food", new LongWritable(10), new BigDecimal("10.5")));
+        builder.close();
+        System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount());
+        System.out.println("Written Row Count: " + builder.getWrittenRowCount());
+    }
+
+    static GTInfo basicInfo() {
+        Builder builder = infoBuilder();
+        GTInfo info = builder.build();
+        return info;
+    }
+
+    static GTInfo advancedInfo() {
+        Builder builder = infoBuilder();
+        builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
+        builder.enableRowBlock(4);
+        GTInfo info = builder.build();
+        return info;
+    }
+
+    private static Builder infoBuilder() {
+        Builder builder = GTInfo.builder();
+        builder.setCodeSystem(new GTSampleCodeSystem());
+        builder.setColumns( //
+                DataType.getInstance("varchar"), //
+                DataType.getInstance("varchar"), //
+                DataType.getInstance("varchar"), //
+                DataType.getInstance("bigint"), //
+                DataType.getInstance("decimal") //
+        );
+        builder.setPrimaryKey(setOf(0));
+        builder.setColumnPreferIndex(setOf(0));
+        return builder;
+    }
+
+    private static BitSet setOf(int... values) {
+        BitSet set = new BitSet();
+        for (int i : values)
+            set.set(i);
+        return set;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7fc2312/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
new file mode 100644
index 0000000..f96b709
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java
@@ -0,0 +1,165 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.serializer.StringSerializer;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class SimpleInvertedIndexTest {
+
+    GTInfo info;
+    GTInvertedIndex index;
+    ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
+    ArrayList<ConciseSet> basicResults = Lists.newArrayList();
+
+    public SimpleInvertedIndexTest() {
+        
+        info = SimpleGridTableTest.advancedInfo();
+        TblColRef colA = info.colRef(0);
+        
+        // block i contains value "i", the last is NULL
+        index = new GTInvertedIndex(info);
+        GTRowBlock mockBlock = GTRowBlock.allocate(info);
+        GTRowBlock.Writer writer = mockBlock.getWriter();
+        GTRecord record = new GTRecord(info);
+        for (int i = 0; i < 10; i++) {
+            record.setValues(i < 9 ? "" + i : null, "", "", new LongWritable(0), new BigDecimal(0));
+            for (int j = 0; j < info.getRowBlockSize(); j++) {
+                writer.append(record);
+            }
+            writer.readyForFlush();
+            index.add(mockBlock);
+            
+            writer.clearForNext();
+        }
+        
+        basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
+        basicResults.add(set(9));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
+        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
+        basicResults.add(set(0));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
+        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
+        basicResults.add(set(0, 5));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
+        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
+        basicResults.add(set(0, 1, 2));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
+        basicResults.add(set(0, 1, 2, 3));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
+        basicResults.add(set(4, 5, 6, 7, 8));
+
+        basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
+        basicResults.add(set(3, 4, 5, 6, 7, 8));
+    }
+
+    @Test
+    public void testBasics() {
+        for (int i = 0; i < basicFilters.size(); i++) {
+            assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
+        }
+    }
+
+    @Test
+    public void testLogicalAnd() {
+        for (int i = 0; i < basicFilters.size(); i++) {
+            for (int j = 0; j < basicFilters.size(); j++) {
+                LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
+                ConciseSet r = basicResults.get(i).clone();
+                r.retainAll(basicResults.get(j));
+                assertEquals(r, index.filter(f));
+            }
+        }
+    }
+
+    @Test
+    public void testLogicalOr() {
+        for (int i = 0; i < basicFilters.size(); i++) {
+            for (int j = 0; j < basicFilters.size(); j++) {
+                LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
+                ConciseSet r = basicResults.get(i).clone();
+                r.addAll(basicResults.get(j));
+                assertEquals(r, index.filter(f));
+            }
+        }
+    }
+
+    @Test
+    public void testNotEvaluable() {
+        ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+        
+        CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
+        assertEquals(all, index.filter(notEvaluable));
+
+        LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
+        assertEquals(all, index.filter(or));
+
+        LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
+        assertEquals(basicResults.get(0), index.filter(and));
+    }
+
+    public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
+        CompareTupleFilter filter = new CompareTupleFilter(op);
+        filter.addChild(columnFilter(col));
+        for (int i : ids) {
+            filter.addChild(constFilter(i));
+        }
+        return filter;
+    }
+
+    public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
+        LogicalTupleFilter filter = new LogicalTupleFilter(op);
+        for (TupleFilter f : filters)
+            filter.addChild(f);
+        return filter;
+    }
+
+    public static ColumnTupleFilter columnFilter(TblColRef col) {
+        return new ColumnTupleFilter(col);
+    }
+
+    public static ConstantTupleFilter constFilter(int id) {
+        byte[] space = new byte[10];
+        ByteBuffer buf = ByteBuffer.wrap(space);
+        StringSerializer stringSerializer = new StringSerializer();
+        stringSerializer.serialize("" + id, buf);
+        ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
+        return new ConstantTupleFilter(data);
+    }
+
+    public static ConciseSet set(int... ints) {
+        ConciseSet set = new ConciseSet();
+        for (int i : ints)
+            set.add(i);
+        return set;
+    }
+
+
+}


Mime
View raw message