kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [1/4] kylin git commit: KYLIN-1453 cuboid sharding based on specific column
Date Fri, 26 Feb 2016 09:55:18 GMT
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 2e1d2f6b6 -> 294fc7078


KYLIN-1453 cuboid sharding based on specific column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294fc707
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294fc707
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294fc707

Branch: refs/heads/2.x-staging
Commit: 294fc70785ab009e6b4d8a12cbeb609d46f89a93
Parents: 3f5074e
Author: honma <honma@ebay.com>
Authored: Thu Feb 25 19:02:28 2016 +0800
Committer: honma <honma@ebay.com>
Committed: Fri Feb 26 17:54:37 2016 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/util/BytesUtil.java |  18 +-
 .../dict/TupleFilterFunctionTransformer.java    | 170 +++++++++++++++++++
 .../dict/TupleFilterFunctionTranslator.java     | 166 ------------------
 .../kylin/metadata/filter/CaseTupleFilter.java  |   7 +-
 .../metadata/filter/ColumnTupleFilter.java      |  11 +-
 .../metadata/filter/CompareTupleFilter.java     |  10 +-
 .../metadata/filter/ConstantTupleFilter.java    |  16 +-
 .../metadata/filter/DynamicTupleFilter.java     |   9 +-
 .../metadata/filter/ExtractTupleFilter.java     |   7 +-
 .../metadata/filter/FunctionTupleFilter.java    |  16 +-
 .../filter/ITupleFilterTransformer.java         |  23 +++
 .../metadata/filter/ITupleFilterTranslator.java |  26 ---
 .../metadata/filter/LogicalTupleFilter.java     |  10 +-
 .../kylin/metadata/filter/TupleFilter.java      |   5 +-
 .../metadata/filter/TupleFilterSerializer.java  |  25 ++-
 .../common/coprocessor/FilterDecorator.java     |   8 +-
 .../hbase/cube/v2/CubeSegmentScanner.java       |  10 +-
 .../common/coprocessor/FilterBaseTest.java      |  46 ++++-
 .../common/coprocessor/FilterEvaluateTest.java  |   4 +-
 .../common/coprocessor/FilterSerializeTest.java |  26 ++-
 20 files changed, 325 insertions(+), 288 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index e01ce4f..4e0701c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -57,7 +57,6 @@ public class BytesUtil {
         return integer;
     }
 
-
     public static long readLong(ByteBuffer buffer, int size) {
         long integer = 0;
         for (int i = 0; i < size; i++) {
@@ -133,11 +132,15 @@ public class BytesUtil {
     // from WritableUtils
     // ============================================================================
 
+
     public static void writeVInt(int i, ByteBuffer out) {
+
         writeVLong(i, out);
+
     }
 
     public static void writeVLong(long i, ByteBuffer out) {
+
         if (i >= -112 && i <= 127) {
             out.put((byte) i);
             return;
@@ -203,6 +206,8 @@ public class BytesUtil {
     }
 
     public static void writeUnsigned(int num, int size, ByteBuffer out) {
+
+
         int mask = 0xff << ((size - 1) * 8);
         for (int i = size; i > 0; i--) {
             int v = (num & mask) >> (i - 1) * 8;
@@ -222,6 +227,7 @@ public class BytesUtil {
     }
 
     public static void writeLong(long num, ByteBuffer out) {
+
         for (int i = 0; i < 8; i++) {
             out.put((byte) num);
             num >>>= 8;
@@ -257,6 +263,8 @@ public class BytesUtil {
         }
         int len = str.length();
         BytesUtil.writeVInt(len, out);
+
+
         for (int i = 0; i < len; i++) {
             out.put((byte) str.charAt(i));
         }
@@ -335,7 +343,7 @@ public class BytesUtil {
             writeVInt(-1, out);
             return;
         }
-        writeVInt(array.length, out);
+        writeVInt(length, out);
         out.put(array, offset, length);
     }
 
@@ -348,7 +356,7 @@ public class BytesUtil {
         in.get(array);
         return array;
     }
-
+    
     public static int peekByteArrayLength(ByteBuffer in) {
         int start = in.position();
         int arrayLen = readVInt(in);
@@ -369,6 +377,7 @@ public class BytesUtil {
         writeVInt(array.length, out);
         byte b_true = (byte) 1;
         byte b_false = (byte) 0;
+
         for (int i = 0; i < array.length; i++) {
             if (array[i])
                 out.put(b_true);
@@ -428,7 +437,4 @@ public class BytesUtil {
         return sb.toString();
     }
 
-    public static void main(String[] args) throws Exception {
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
new file mode 100644
index 0000000..096a28d
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import java.util.Collection;
+import java.util.ListIterator;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FunctionTupleFilter;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+/**
+ * only take effect when the compare filter has function
+ */
+public class TupleFilterFunctionTransformer implements ITupleFilterTransformer {
+    public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTransformer.class);
+
+    private IDictionaryAware dictionaryAware;
+
+    public TupleFilterFunctionTransformer(IDictionaryAware dictionaryAware) {
+        this.dictionaryAware = dictionaryAware;
+    }
+
+    @Override
+    public TupleFilter transform(TupleFilter tupleFilter) {
+        TupleFilter translated = null;
+        if (tupleFilter instanceof CompareTupleFilter) {
+            translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+            if (translated != null) {
+                logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+            }
+        } else if (tupleFilter instanceof FunctionTupleFilter) {
+            translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
+            if (translated != null) {
+                logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+            }
+        } else if (tupleFilter instanceof LogicalTupleFilter) {
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter transformed = transform(childIterator.next());
+                if (transformed != null)
+                    childIterator.set(transformed);
+            }
+        }
+        return translated == null ? tupleFilter : translated;
+    }
+
+    private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
+        if (!functionTupleFilter.isValid())
+            return null;
+
+        TblColRef columnRef = functionTupleFilter.getColumn();
+        Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+        if (dict == null)
+            return null;
+
+        CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+        translated.addChild(new ColumnTupleFilter(columnRef));
+
+        try {
+            for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+                Object dictVal = dict.getValueFromId(i);
+                if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
+                    translated.addChild(new ConstantTupleFilter(dictVal));
+                }
+            }
+        } catch (Exception e) {
+            logger.debug(e.getMessage());
+            return null;
+        }
+        return translated;
+    }
+
+    @SuppressWarnings("unchecked")
+    private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
+        if (compTupleFilter.getFunction() == null)
+            return null;
+
+        FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
+        if (!functionTupleFilter.isValid())
+            return null;
+
+        TblColRef columnRef = functionTupleFilter.getColumn();
+        Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+        if (dict == null)
+            return null;
+
+        CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+        translated.addChild(new ColumnTupleFilter(columnRef));
+
+        try {
+            Collection<Object> inValues = Lists.newArrayList();
+            for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+                Object dictVal = dict.getValueFromId(i);
+                Object computedVal = functionTupleFilter.invokeFunction(dictVal);
+                Class clazz = Primitives.wrap(computedVal.getClass());
+                Object targetVal = compTupleFilter.getFirstValue();
+                if (Primitives.isWrapperType(clazz))
+                    targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+                int comp = ((Comparable) computedVal).compareTo(targetVal);
+                boolean compResult = false;
+                switch (compTupleFilter.getOperator()) {
+                case EQ:
+                    compResult = comp == 0;
+                    break;
+                case NEQ:
+                    compResult = comp != 0;
+                    break;
+                case LT:
+                    compResult = comp < 0;
+                    break;
+                case LTE:
+                    compResult = comp <= 0;
+                    break;
+                case GT:
+                    compResult = comp > 0;
+                    break;
+                case GTE:
+                    compResult = comp >= 0;
+                    break;
+                case IN:
+                    compResult = compTupleFilter.getValues().contains(computedVal.toString());
+                    break;
+                case NOTIN:
+                    compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+                    break;
+                default:
+                    break;
+                }
+                if (compResult) {
+                    inValues.add(dictVal);
+                }
+            }
+            translated.addChild(new ConstantTupleFilter(inValues));
+        } catch (Exception e) {
+            logger.debug(e.getMessage());
+            return null;
+        }
+        return translated;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
deleted file mode 100644
index 1c96dd4..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.util.ListIterator;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.primitives.Primitives;
-
-/**
- * only take effect when the compare filter has function
- */
-public class TupleFilterFunctionTranslator implements ITupleFilterTranslator {
-    public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTranslator.class);
-
-    private IDictionaryAware dictionaryAware;
-
-    public TupleFilterFunctionTranslator(IDictionaryAware dictionaryAware) {
-        this.dictionaryAware = dictionaryAware;
-    }
-
-    @Override
-    public TupleFilter translate(TupleFilter tupleFilter) {
-        TupleFilter translated = null;
-        if (tupleFilter instanceof CompareTupleFilter) {
-            translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
-            if (translated != null) {
-                logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
-            }
-        } else if (tupleFilter instanceof FunctionTupleFilter) {
-            translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
-            if (translated != null) {
-                logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
-            }
-        } else if (tupleFilter instanceof LogicalTupleFilter) {
-            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
-            while (childIterator.hasNext()) {
-                TupleFilter tempTranslated = translate(childIterator.next());
-                if (tempTranslated != null)
-                    childIterator.set(tempTranslated);
-            }
-        }
-        return translated == null ? tupleFilter : translated;
-    }
-
-    private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
-        if (!functionTupleFilter.isValid())
-            return null;
-
-        TblColRef columnRef = functionTupleFilter.getColumn();
-        Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
-        if (dict == null)
-            return null;
-
-        CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
-        translated.addChild(new ColumnTupleFilter(columnRef));
-
-        try {
-            for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
-                Object dictVal = dict.getValueFromId(i);
-                if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
-                    translated.addChild(new ConstantTupleFilter(dictVal));
-                }
-            }
-        } catch (Exception e) {
-            logger.debug(e.getMessage());
-            return null;
-        }
-        return translated;
-    }
-
-    @SuppressWarnings("unchecked")
-    private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
-        if (compTupleFilter.getFunction() == null)
-            return null;
-
-        FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
-        if (!functionTupleFilter.isValid())
-            return null;
-
-        TblColRef columnRef = functionTupleFilter.getColumn();
-        Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
-        if (dict == null)
-            return null;
-
-        CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
-        translated.addChild(new ColumnTupleFilter(columnRef));
-
-        try {
-            for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
-                Object dictVal = dict.getValueFromId(i);
-                Object computedVal = functionTupleFilter.invokeFunction(dictVal);
-                Class clazz = Primitives.wrap(computedVal.getClass());
-                Object targetVal = compTupleFilter.getFirstValue();
-                if (Primitives.isWrapperType(clazz))
-                    targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
-
-                int comp = ((Comparable) computedVal).compareTo(targetVal);
-                boolean compResult = false;
-                switch (compTupleFilter.getOperator()) {
-                case EQ:
-                    compResult = comp == 0;
-                    break;
-                case NEQ:
-                    compResult = comp != 0;
-                    break;
-                case LT:
-                    compResult = comp < 0;
-                    break;
-                case LTE:
-                    compResult = comp <= 0;
-                    break;
-                case GT:
-                    compResult = comp > 0;
-                    break;
-                case GTE:
-                    compResult = comp >= 0;
-                    break;
-                case IN:
-                    compResult = compTupleFilter.getValues().contains(computedVal.toString());
-                    break;
-                case NOTIN:
-                    compResult = !compTupleFilter.getValues().contains(computedVal.toString());
-                    break;
-                default:
-                    break;
-                }
-                if (compResult) {
-                    translated.addChild(new ConstantTupleFilter(dictVal));
-                }
-            }
-        } catch (Exception e) {
-            logger.debug(e.getMessage());
-            return null;
-        }
-        return translated;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 6dbc614..2b00d69 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.metadata.filter;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -101,12 +102,12 @@ public class CaseTupleFilter extends TupleFilter {
     }
 
     @Override
-    public byte[] serialize(IFilterCodeSystem<?> cs) {
-        return new byte[0];
+    public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+        //serialize nothing
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 0d2a73d..029233d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -84,8 +84,7 @@ public class ColumnTupleFilter extends TupleFilter {
     }
 
     @Override
-    public byte[] serialize(IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+    public void serialize(IFilterCodeSystem<?> cs,ByteBuffer buffer) {
         String table = columnRef.getTable();
         BytesUtil.writeUTFString(table, buffer);
 
@@ -97,17 +96,13 @@ public class ColumnTupleFilter extends TupleFilter {
 
         String dataType = columnRef.getDatatype();
         BytesUtil.writeUTFString(dataType, buffer);
-
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
         TableDesc table = null;
         ColumnDesc column = new ColumnDesc();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
 
         String tableName = BytesUtil.readUTFString(buffer);
         if (tableName != null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 248ab3b..fc0bab7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -214,23 +214,19 @@ public class CompareTupleFilter extends TupleFilter {
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     @Override
-    public byte[] serialize(IFilterCodeSystem cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+    public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
         int size = this.dynamicVariables.size();
         BytesUtil.writeVInt(size, buffer);
         for (Map.Entry<String, Object> entry : this.dynamicVariables.entrySet()) {
             BytesUtil.writeUTFString(entry.getKey(), buffer);
             cs.serialize(entry.getValue(), buffer);
         }
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
         this.dynamicVariables.clear();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
         int size = BytesUtil.readVInt(buffer);
         for (int i = 0; i < size; i++) {
             String name = BytesUtil.readUTFString(buffer);

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 3056a9c..db3eb4f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -21,8 +21,10 @@ package org.apache.kylin.metadata.filter;
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.TreeSet;
 
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.comparators.NullComparator;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
 
@@ -40,7 +42,7 @@ public class ConstantTupleFilter extends TupleFilter {
 
     public ConstantTupleFilter() {
         super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
-        this.constantValues = new HashSet<Object>();
+        this.constantValues = Lists.newArrayList();
     }
 
     public ConstantTupleFilter(Object value) {
@@ -89,22 +91,18 @@ public class ConstantTupleFilter extends TupleFilter {
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     @Override
-    public byte[] serialize(IFilterCodeSystem cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+    public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
         int size = this.constantValues.size();
         BytesUtil.writeVInt(size, buffer);
         for (Object val : this.constantValues) {
             cs.serialize(val, buffer);
         }
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
         this.constantValues.clear();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
         int size = BytesUtil.readVInt(buffer);
         for (int i = 0; i < size; i++) {
             this.constantValues.add(cs.deserialize(buffer));

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index a482519..d9dc52a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -69,17 +69,12 @@ public class DynamicTupleFilter extends TupleFilter {
     }
 
     @Override
-    public byte[] serialize(IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+    public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
         BytesUtil.writeUTFString(variableName, buffer);
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
         this.variableName = BytesUtil.readUTFString(buffer);
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index 6f7dfaf..591e64b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.metadata.filter;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -113,12 +114,12 @@ public class ExtractTupleFilter extends TupleFilter {
     }
 
     @Override
-    public byte[] serialize(IFilterCodeSystem<?> cs) {
-        return new byte[0];
+    public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+        //do nothing
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
index 30bef97..2a08728 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java
@@ -123,32 +123,20 @@ public class FunctionTupleFilter extends TupleFilter {
     }
 
     @Override
-    byte[] serialize(IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+    void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
         BytesUtil.writeUTFString(name, buffer);
         BytesUtil.writeVInt(colPosition, buffer);
         BytesUtil.writeVInt(isValid ? 1 : 0, buffer);
-        BytesUtil.writeByteArray(TupleFilterSerializer.serialize(columnContainerFilter, cs), buffer);
-
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
     }
 
     @Override
-    void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
 
         this.name = BytesUtil.readUTFString(buffer);
         this.initMethod();
 
         this.colPosition = BytesUtil.readVInt(buffer);
         this.isValid = BytesUtil.readVInt(buffer) == 1;
-
-        byte[] columnFilter = BytesUtil.readByteArray(buffer);
-        if (columnFilter != null) {
-            this.columnContainerFilter = TupleFilterSerializer.deserialize(columnFilter, cs);
-        }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
new file mode 100644
index 0000000..d3d5076
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+public interface ITupleFilterTransformer {
+    TupleFilter transform(TupleFilter tupleFilter);
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
deleted file mode 100644
index aed284c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public interface ITupleFilterTranslator {
-    TupleFilter translate(TupleFilter tupleFilter);
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 0929cf1..61657fb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.metadata.filter;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -46,7 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
         return cloneTuple;
     }
 
-
     //    private TupleFilter reverseNestedNots(TupleFilter filter, int depth) {
     //        if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT)) {
     //            assert (filter.children.size() == 1);
@@ -60,7 +60,6 @@ public class LogicalTupleFilter extends TupleFilter {
     //        }
     //    }
 
-
     @Override
     public TupleFilter reverse() {
         switch (operator) {
@@ -151,12 +150,13 @@ public class LogicalTupleFilter extends TupleFilter {
     }
 
     @Override
-    public byte[] serialize(IFilterCodeSystem<?> cs) {
-        return new byte[0];
+    public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+        //do nothing
     }
 
     @Override
-    public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 944ddd0..1e23499 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.metadata.filter;
 
+import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -204,9 +205,9 @@ public abstract class TupleFilter {
 
     public abstract Collection<?> getValues();
 
-    abstract byte[] serialize(IFilterCodeSystem<?> cs);
+    abstract void serialize(IFilterCodeSystem<?> cs,ByteBuffer buffer);
 
-    abstract void deserialize(byte[] bytes, IFilterCodeSystem<?> cs);
+    abstract void deserialize(IFilterCodeSystem<?> cs,ByteBuffer buffer);
 
     public static boolean isEvaluableRecursively(TupleFilter filter) {
         if (filter == null)

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index a394a51..39ccb15 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -18,12 +18,15 @@
 
 package org.apache.kylin.metadata.filter;
 
+import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Stack;
 
 import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
@@ -33,6 +36,8 @@ import org.apache.kylin.common.util.BytesUtil;
  */
 public class TupleFilterSerializer {
 
+    private static final Logger logger = LoggerFactory.getLogger(TupleFilterSerializer.class);
+
     public interface Decorator {
         TupleFilter onSerialize(TupleFilter filter);
     }
@@ -51,8 +56,18 @@ public class TupleFilterSerializer {
     }
 
     public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem<?> cs) {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-        internalSerialize(rootFilter, decorator, buffer, cs);
+        ByteBuffer buffer;
+        int bufferSize = BUFFER_SIZE;
+        while (true) {
+            try {
+                buffer = ByteBuffer.allocate(bufferSize);
+                internalSerialize(rootFilter, decorator, buffer, cs);
+                break;
+            } catch (BufferOverflowException e) {
+                logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize);
+                bufferSize *= 4;
+            }
+        }
         byte[] result = new byte[buffer.position()];
         System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
         return result;
@@ -86,10 +101,9 @@ public class TupleFilterSerializer {
         if (flag < 0) {
             BytesUtil.writeVInt(-1, buffer);
         } else {
-            byte[] bytes = filter.serialize(cs);
             int opVal = filter.getOperator().getValue();
             BytesUtil.writeVInt(opVal, buffer);
-            BytesUtil.writeByteArray(bytes, buffer);
+            filter.serialize(cs, buffer);
             BytesUtil.writeVInt(flag, buffer);
         }
     }
@@ -107,8 +121,7 @@ public class TupleFilterSerializer {
 
             // deserialize filter
             TupleFilter filter = createTupleFilter(opVal);
-            byte[] filterBytes = BytesUtil.readByteArray(buffer);
-            filter.deserialize(filterBytes, cs);
+            filter.deserialize(cs, buffer);
 
             if (rootFilter == null) {
                 // push root to stack

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 01d3041..5208ba7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -8,11 +8,11 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.kv.RowKeyColumnIO;
 import org.apache.kylin.dict.DictCodeSystem;
 import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.dict.TupleFilterFunctionTranslator;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilterSerializer;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -131,8 +131,8 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
         if (filter == null)
             return null;
 
-        ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(columnIO.getIDictionaryAware());
-        filter = translator.translate(filter);
+        ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(columnIO.getIDictionaryAware());
+        filter = translator.transform(filter);
 
         // un-evaluatable filter is replaced with TRUE
         if (!filter.isEvaluable()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index e96c602..3f00566 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -23,7 +23,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable;
 import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
 import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.TupleFilterFunctionTranslator;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
 import org.apache.kylin.gridtable.EmptyGTScanner;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.GTRecord;
@@ -33,7 +33,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.GTUtil;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -63,9 +63,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
         CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
 
-        // translate FunctionTupleFilter to IN clause
-        ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(this.cubeSeg);
-        filter = translator.translate(filter);
+        // transform FunctionTupleFilter to equivalent IN clause
+        ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(this.cubeSeg);
+        filter = translator.transform(filter);
 
         //replace the constant values in filter to dictionary codes 
         TupleFilter gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, mapping.getCuboidDimensionsInGTOrder(), groups);

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
index 92e9699..e7ed1a8 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -18,8 +18,12 @@
 
 package org.apache.kylin.storage.hbase.common.coprocessor;
 
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.Random;
 
@@ -39,6 +43,8 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.Tuple;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 
+import com.google.common.collect.Lists;
+
 /**
  * @author xjiang
  * 
@@ -64,7 +70,7 @@ public class FilterBaseTest {
         return groups;
     }
 
-    protected CompareTupleFilter buildCompareFilter(List<TblColRef> groups, int index) {
+    protected CompareTupleFilter buildEQCompareFilter(List<TblColRef> groups, int index) {
         TblColRef column = groups.get(index);
         CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
         ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
@@ -79,9 +85,31 @@ public class FilterBaseTest {
         return compareFilter;
     }
 
+    protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
+        CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
+        ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
+        compareFilter.addChild(columnFilter);
+
+        List<String> inValues = Lists.newArrayList();
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+        Date startDate = simpleDateFormat.parse("1970-01-01");
+        Date endDate = simpleDateFormat.parse("2100-01-01");
+        Calendar start = Calendar.getInstance();
+        start.setTime(startDate);
+        Calendar end = Calendar.getInstance();
+        end.setTime(endDate);
+        for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
+            inValues.add(simpleDateFormat.format(date));
+        }
+
+        ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
+        compareFilter.addChild(constantFilter);
+        return compareFilter;
+    }
+
     protected TupleFilter buildAndFilter(List<TblColRef> columns) {
-        CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
-        CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+        CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
+        CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
         LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
         andFilter.addChild(compareFilter1);
         andFilter.addChild(compareFilter2);
@@ -89,8 +117,8 @@ public class FilterBaseTest {
     }
 
     protected TupleFilter buildOrFilter(List<TblColRef> columns) {
-        CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
-        CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+        CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
+        CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
         LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
         logicFilter.addChild(compareFilter1);
         logicFilter.addChild(compareFilter2);
@@ -105,12 +133,12 @@ public class FilterBaseTest {
         TupleFilter then0 = new ConstantTupleFilter("0");
         caseFilter.addChild(then0);
 
-        TupleFilter when1 = buildCompareFilter(groups, 0);
+        TupleFilter when1 = buildEQCompareFilter(groups, 0);
         caseFilter.addChild(when1);
         TupleFilter then1 = new ConstantTupleFilter("1");
         caseFilter.addChild(then1);
 
-        TupleFilter when2 = buildCompareFilter(groups, 1);
+        TupleFilter when2 = buildEQCompareFilter(groups, 1);
         caseFilter.addChild(when2);
         TupleFilter then2 = new ConstantTupleFilter("2");
         caseFilter.addChild(then2);
@@ -153,9 +181,9 @@ public class FilterBaseTest {
         }
 
         String str1 = f1.toString();
-        System.out.println("f1=" + str1);
+        //System.out.println("f1=" + str1);
         String str2 = f2.toString();
-        System.out.println("f2=" + str2);
+        //System.out.println("f2=" + str2);
         if (!str1.equals(str2)) {
             throw new IllegalStateException("f1=" + str1 + ", f2=" + str2);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
index bde8dd2..aac09b7 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -38,7 +38,7 @@ public class FilterEvaluateTest extends FilterBaseTest {
     @Test
     public void testEvaluate00() {
         List<TblColRef> groups = buildGroups();
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -54,7 +54,7 @@ public class FilterEvaluateTest extends FilterBaseTest {
     @Test
     public void testEvaluate01() {
         List<TblColRef> groups = buildGroups();
-        TupleFilter filter = buildCompareFilter(groups, 1);
+        TupleFilter filter = buildEQCompareFilter(groups, 1);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
index 8bf8ecb..0a21598 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.storage.hbase.common.coprocessor;
 
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -40,7 +41,7 @@ public class FilterSerializeTest extends FilterBaseTest {
     @Test
     public void testSerialize01() {
         List<TblColRef> groups = buildGroups();
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -51,7 +52,7 @@ public class FilterSerializeTest extends FilterBaseTest {
     @Test
     public void testSerialize02() {
         List<TblColRef> groups = buildGroups();
-        TupleFilter filter = buildCompareFilter(groups, 1);
+        TupleFilter filter = buildEQCompareFilter(groups, 1);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -88,7 +89,7 @@ public class FilterSerializeTest extends FilterBaseTest {
         TblColRef colRef = new TblColRef(column);
         List<TblColRef> groups = new ArrayList<TblColRef>();
         groups.add(colRef);
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -103,7 +104,7 @@ public class FilterSerializeTest extends FilterBaseTest {
         TblColRef colRef = new TblColRef(column);
         List<TblColRef> groups = new ArrayList<TblColRef>();
         groups.add(colRef);
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -122,7 +123,7 @@ public class FilterSerializeTest extends FilterBaseTest {
         TblColRef colRef = new TblColRef(column);
         List<TblColRef> groups = new ArrayList<TblColRef>();
         groups.add(colRef);
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -140,7 +141,7 @@ public class FilterSerializeTest extends FilterBaseTest {
         TblColRef colRef = new TblColRef(column);
         List<TblColRef> groups = new ArrayList<TblColRef>();
         groups.add(colRef);
-        TupleFilter filter = buildCompareFilter(groups, 0);
+        TupleFilter filter = buildEQCompareFilter(groups, 0);
 
         byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
         TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
@@ -203,6 +204,19 @@ public class FilterSerializeTest extends FilterBaseTest {
     }
 
     @Test
+    public void testSerialize14() throws ParseException {
+        List<TblColRef> groups = buildGroups();
+        TupleFilter filter = buildINCompareFilter(groups.get(0));
+
+        long start = System.currentTimeMillis();
+        byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+        System.out.println("Size of serialized filter " + bytes.length + ", serialize time: " + (System.currentTimeMillis() - start));
+        TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+        compareFilter(filter, newFilter);
+    }
+
+    @Test
     public void testDynamic() {
         final CompareTupleFilter compareDynamicFilter = buildCompareDynamicFilter(buildGroups());
 


Mime
View raw message