kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From luke...@apache.org
Subject [08/50] [abbrv] incubator-kylin git commit: put tuplefilter and Ituple into medadata module
Date Thu, 15 Jan 2015 12:26:24 GMT
put tuplefilter and Ituple into medadata module


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/94352aec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/94352aec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/94352aec

Branch: refs/heads/inverted-index
Commit: 94352aec8b9c57366543b24e58e837b769d4f6d8
Parents: 41b327d
Author: honma <honma@ebay.com>
Authored: Mon Jan 12 21:44:46 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Tue Jan 13 10:07:15 2015 +0800

----------------------------------------------------------------------
 .../java/com/kylinolap/cube/CubeInstance.java   |   1 -
 .../com/kylinolap/invertedindex/IIInstance.java |   4 -
 .../job/coprocessor/IIEndpointTest.java         |  10 +-
 .../metadata/filter/CaseTupleFilter.java        | 110 +++++++++
 .../metadata/filter/ColumnTupleFilter.java      | 116 +++++++++
 .../metadata/filter/CompareTupleFilter.java     | 240 +++++++++++++++++++
 .../metadata/filter/ConstantTupleFilter.java    | 101 ++++++++
 .../metadata/filter/DynamicTupleFilter.java     |  83 +++++++
 .../metadata/filter/ExtractTupleFilter.java     | 107 +++++++++
 .../metadata/filter/LogicalTupleFilter.java     | 125 ++++++++++
 .../kylinolap/metadata/filter/TupleFilter.java  | 207 ++++++++++++++++
 .../metadata/filter/TupleFilterSerializer.java  | 179 ++++++++++++++
 .../metadata/realization/IRealization.java      |   2 -
 .../metadata/realization/SQLDigest.java         |   2 +-
 .../com/kylinolap/metadata/tuple/ITuple.java    |  41 ++++
 .../metadata/tuple/ITupleIterator.java          |  44 ++++
 .../query/enumerator/CubeEnumerator.java        |   8 +-
 .../kylinolap/query/relnode/OLAPContext.java    |   5 +-
 .../kylinolap/query/relnode/OLAPFilterRel.java  |  18 +-
 .../com/kylinolap/storage/IStorageEngine.java   |   2 +-
 .../storage/filter/BitMapFilterEvaluator.java   |   3 +
 .../storage/filter/CaseTupleFilter.java         | 109 ---------
 .../storage/filter/ColumnTupleFilter.java       | 116 ---------
 .../storage/filter/CompareTupleFilter.java      | 240 -------------------
 .../storage/filter/ConstantTupleFilter.java     | 101 --------
 .../storage/filter/DynamicTupleFilter.java      |  83 -------
 .../storage/filter/ExtractTupleFilter.java      | 106 --------
 .../storage/filter/LogicalTupleFilter.java      | 124 ----------
 .../kylinolap/storage/filter/TupleFilter.java   | 207 ----------------
 .../storage/filter/TupleFilterSerializer.java   | 180 --------------
 .../storage/hbase/ColumnValueRange.java         |   2 +-
 .../hbase/ConcurrentHBaseTupleIterator.java     |   4 +-
 .../storage/hbase/CubeSegmentTupleIterator.java |   4 +-
 .../storage/hbase/CubeStorageEngine.java        |  12 +-
 .../storage/hbase/DerivedFilterTranslator.java  |  14 +-
 .../hbase/InvertedIndexStorageEngine.java       |   2 +-
 .../hbase/SerializedHBaseTupleIterator.java     |   6 +-
 .../hbase/coprocessor/CoprocessorFilter.java    |  16 +-
 .../endpoint/EndpointTupleIterator.java         |   8 +-
 .../coprocessor/observer/ObserverEnabler.java   |   2 +-
 .../coprocessor/observer/ObserverTuple.java     |   2 +-
 .../com/kylinolap/storage/tuple/ITuple.java     |  41 ----
 .../kylinolap/storage/tuple/ITupleIterator.java |  44 ----
 .../java/com/kylinolap/storage/tuple/Tuple.java |   1 +
 .../filter/BitMapFilterEvaluatorTest.java       |   6 +-
 .../storage/filter/FilterBaseTest.java          |   3 +-
 .../storage/filter/FilterEvaluateTest.java      |   2 +
 .../storage/filter/FilterSerializeTest.java     |   5 +-
 .../endpoint/EndpointAggregationTest.java       |   8 +-
 .../com/kylinolap/storage/test/StorageTest.java |  21 +-
 50 files changed, 1442 insertions(+), 1435 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
index 3b38a14..f1d3953 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
@@ -417,7 +417,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
         return Lists.newArrayList(getDescriptor().listAllColumns());
     }
 
-    @Override
     public List<TblColRef> getDimensions() {
         List<TblColRef> ret = Lists.newArrayList();
         for (DimensionDesc dim : getDescriptor().getDimensions()) {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
index f77d5ef..53f5a0c 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
@@ -409,8 +409,4 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
         return getDescriptor().getMeasures();
     }
 
-    @Override
-    public List<TblColRef> getDimensions() {
-        throw new UnsupportedOperationException();
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java b/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
index 26ede05..adede2d 100644
--- a/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
+++ b/job/src/test/java/com/kylinolap/job/coprocessor/IIEndpointTest.java
@@ -23,12 +23,12 @@ import com.kylinolap.invertedindex.model.IIKeyValueCodec;
 import com.kylinolap.metadata.MetadataManager;
 import com.kylinolap.metadata.model.*;
 import com.kylinolap.storage.StorageContext;
-import com.kylinolap.storage.filter.ColumnTupleFilter;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.ConstantTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
+import com.kylinolap.metadata.filter.ColumnTupleFilter;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.ConstantTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
 import com.kylinolap.storage.hbase.coprocessor.endpoint.EndpointTupleIterator;
-import com.kylinolap.storage.tuple.ITuple;
+import com.kylinolap.metadata.tuple.ITuple;
 
 /**
  * Created by Hongbin Ma(Binmahone) on 11/14/14.

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
new file mode 100644
index 0000000..d40dd4a
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * @author xjiang
+ * 
+ */
+public class CaseTupleFilter extends TupleFilter {
+
+    private List<TupleFilter> whenFilters;
+    private List<TupleFilter> thenFilters;
+    private TupleFilter elseFilter;
+    private Collection<String> values;
+    private int filterIndex;
+
+    public CaseTupleFilter() {
+        super(new ArrayList<TupleFilter>(), FilterOperatorEnum.CASE);
+        this.filterIndex = 0;
+        this.values = Collections.emptyList();
+        this.whenFilters = new ArrayList<TupleFilter>();
+        this.thenFilters = new ArrayList<TupleFilter>();
+        this.elseFilter = null;
+    }
+
+    @Override
+    public void addChild(TupleFilter child) {
+        super.addChild(child);
+        if (this.filterIndex % 2 == 0) {
+            this.whenFilters.add(child);
+        } else {
+            this.thenFilters.add(child);
+        }
+        this.filterIndex++;
+    }
+
+    @Override
+    public String toString() {
+        return "CaseTupleFilter [when=" + whenFilters + ", then=" + thenFilters + ", else=" + elseFilter + ", children=" + children + "]";
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        if (whenFilters.size() != thenFilters.size()) {
+            elseFilter = whenFilters.remove(whenFilters.size() - 1);
+        }
+        boolean matched = false;
+        for (int i = 0; i < whenFilters.size(); i++) {
+            TupleFilter whenFilter = whenFilters.get(i);
+            if (whenFilter.evaluate(tuple)) {
+                TupleFilter thenFilter = thenFilters.get(i);
+                thenFilter.evaluate(tuple);
+                values = thenFilter.getValues();
+                matched = true;
+                break;
+            }
+        }
+        if (!matched) {
+            if (elseFilter != null) {
+                elseFilter.evaluate(tuple);
+                values = elseFilter.getValues();
+            } else {
+                values = Collections.emptyList();
+            }
+        }
+
+        return true;
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return false;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        return this.values;
+    }
+
+    @Override
+    public byte[] serialize() {
+        return new byte[0];
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
new file mode 100644
index 0000000..3011425
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ColumnTupleFilter.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.model.ColumnDesc;
+import com.kylinolap.metadata.model.TableDesc;
+import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public class ColumnTupleFilter extends TupleFilter {
+
+    private TblColRef columnRef;
+    private Object tupleValue;
+    private List<String> values;
+
+    public ColumnTupleFilter(TblColRef column) {
+        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
+        this.columnRef = column;
+        this.values = new ArrayList<String>(1);
+        this.values.add(null);
+    }
+
+    public TblColRef getColumn() {
+        return columnRef;
+    }
+
+    public void setColumn(TblColRef col) {
+        this.columnRef = col;
+    }
+
+    @Override
+    public void addChild(TupleFilter child) {
+        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+    }
+
+    @Override
+    public String toString() {
+        return "ColumnFilter [column=" + columnRef + "]";
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        this.tupleValue = tuple.getValue(columnRef);
+        return true;
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return true;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        this.values.set(0, (String) this.tupleValue);
+        return this.values;
+    }
+
+    @Override
+    public byte[] serialize() {
+        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+        String table = columnRef.getTable();
+        BytesUtil.writeUTFString(table, buffer);
+
+        String columnName = columnRef.getName();
+        BytesUtil.writeUTFString(columnName, buffer);
+
+        String dataType = columnRef.getDatatype();
+        BytesUtil.writeUTFString(dataType, buffer);
+
+        byte[] result = new byte[buffer.position()];
+        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+        return result;
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+        ColumnDesc column = new ColumnDesc();
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        String tableName = BytesUtil.readUTFString(buffer);
+        if (tableName != null) {
+            TableDesc table = new TableDesc();
+            table.setName(tableName);
+            column.setTable(table);
+        }
+
+        column.setName(BytesUtil.readUTFString(buffer));
+        column.setDatatype(BytesUtil.readUTFString(buffer));
+
+        this.columnRef = new TblColRef(column);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
new file mode 100644
index 0000000..9c3c294
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/CompareTupleFilter.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * @author xjiang
+ */
+public class CompareTupleFilter extends TupleFilter {
+
+    private TblColRef column;
+    private Collection<String> conditionValues;
+    private String firstCondValue;
+    private Map<String, String> dynamicVariables;
+    private String nullString;
+
+    public CompareTupleFilter(FilterOperatorEnum op) {
+        super(new ArrayList<TupleFilter>(2), op);
+        this.conditionValues = new HashSet<String>();
+        this.dynamicVariables = new HashMap<String, String>();
+        boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
+                || op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
+                || op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
+                || op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN //
+                || op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL);
+        if (opGood == false)
+            throw new IllegalArgumentException("Unsupported operator " + op);
+    }
+
+    private CompareTupleFilter(CompareTupleFilter another) {
+        super(new ArrayList<TupleFilter>(another.children), another.operator);
+        this.column = another.column;
+        this.conditionValues = new HashSet<String>();
+        this.conditionValues.addAll(another.conditionValues);
+        this.dynamicVariables = new HashMap<String, String>();
+        this.dynamicVariables.putAll(another.dynamicVariables);
+    }
+
+    @Override
+    public void addChild(TupleFilter child) {
+        super.addChild(child);
+        if (child instanceof ColumnTupleFilter) {
+            ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
+            if (this.column != null) {
+                throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
+            }
+            this.column = columnFilter.getColumn();
+            // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
+            if (!this.conditionValues.isEmpty() && needSwapOperator()) {
+                this.operator = SWAP_OP_MAP.get(this.operator);
+            }
+        } else if (child instanceof ConstantTupleFilter) {
+            this.conditionValues.addAll(child.getValues());
+            this.firstCondValue = this.conditionValues.iterator().next();
+        } else if (child instanceof DynamicTupleFilter) {
+            DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
+            this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
+        }
+        //TODO
+        //        else if (child instanceof ExtractTupleFilter) {
+        //        } else if (child instanceof CaseTupleFilter) {
+        //        }
+    }
+
+    private boolean needSwapOperator() {
+        return operator == FilterOperatorEnum.LT || operator == FilterOperatorEnum.GT || operator == FilterOperatorEnum.LTE || operator == FilterOperatorEnum.GTE;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        return conditionValues;
+    }
+
+    public String getFirstValue() {
+        return firstCondValue;
+    }
+
+    public TblColRef getColumn() {
+        return column;
+    }
+
+    public Map<String, String> getVariables() {
+        return dynamicVariables;
+    }
+
+    public void bindVariable(String variable, String value) {
+        this.dynamicVariables.put(variable, value);
+        this.conditionValues.add(value);
+        this.firstCondValue = this.conditionValues.iterator().next();
+    }
+
+    public String getNullString() {
+        return nullString;
+    }
+
+    public void setNullString(String nullString) {
+        this.nullString = nullString;
+    }
+
+    @Override
+    public TupleFilter copy() {
+        return new CompareTupleFilter(this);
+    }
+
+    @Override
+    public TupleFilter reverse() {
+        TupleFilter reverse = copy();
+        reverse.operator = REVERSE_OP_MAP.get(this.operator);
+        return reverse;
+    }
+
+    @Override
+    public String toString() {
+        return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
+    }
+
+    // TODO requires generalize, currently only evaluates COLUMN {op} CONST
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        // extract tuple value
+        String tupleValue = null;
+        for (TupleFilter filter : this.children) {
+            if (isConstant(filter) == false) {
+                filter.evaluate(tuple);
+                tupleValue = filter.getValues().iterator().next();
+            }
+        }
+
+        // consider null string
+        if (nullString != null && nullString.equals(tupleValue)) {
+            tupleValue = null;
+        }
+        if (tupleValue == null) {
+            if (operator == FilterOperatorEnum.ISNULL)
+                return true;
+            else
+                return false;
+        }
+
+        // always false if compare to null
+        if (firstCondValue.equals(nullString))
+            return false;
+
+        // tricky here -- order is ensured by string compare (even for number columns)
+        // because it's row key ID (not real value) being compared
+        int comp = tupleValue.compareTo(firstCondValue);
+
+        boolean result;
+        switch (operator) {
+        case EQ:
+            result = comp == 0;
+            break;
+        case NEQ:
+            result = comp != 0;
+            break;
+        case LT:
+            result = comp < 0;
+            break;
+        case LTE:
+            result = comp <= 0;
+            break;
+        case GT:
+            result = comp > 0;
+            break;
+        case GTE:
+            result = comp >= 0;
+            break;
+        case IN:
+            result = conditionValues.contains(tupleValue);
+            break;
+        case NOTIN:
+            result = !conditionValues.contains(tupleValue);
+            break;
+        default:
+            result = false;
+        }
+        return result;
+    }
+
+    private boolean isConstant(TupleFilter filter) {
+        return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return column != null && !conditionValues.isEmpty();
+    }
+
+    @Override
+    public byte[] serialize() {
+        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+        int size = this.dynamicVariables.size();
+        BytesUtil.writeVInt(size, buffer);
+        for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
+            BytesUtil.writeUTFString(entry.getKey(), buffer);
+            BytesUtil.writeUTFString(entry.getValue(), buffer);
+        }
+        BytesUtil.writeAsciiString(nullString, buffer);
+        byte[] result = new byte[buffer.position()];
+        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+        return result;
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+        this.dynamicVariables.clear();
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        int size = BytesUtil.readVInt(buffer);
+        for (int i = 0; i < size; i++) {
+            String nameString = BytesUtil.readUTFString(buffer);
+            String valueString = BytesUtil.readUTFString(buffer);
+            bindVariable(nameString, valueString);
+        }
+        this.nullString = BytesUtil.readAsciiString(buffer);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
new file mode 100644
index 0000000..b18a49c
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ConstantTupleFilter.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public class ConstantTupleFilter extends TupleFilter {
+
+    public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
+    public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
+
+    private Collection<String> constantValues;
+
+    public ConstantTupleFilter() {
+        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
+        this.constantValues = new HashSet<String>();
+    }
+
+    public ConstantTupleFilter(String value) {
+        this();
+        this.constantValues.add(value);
+    }
+
+    public ConstantTupleFilter(Collection<String> values) {
+        this();
+        this.constantValues.addAll(values);
+    }
+
+    @Override
+    public void addChild(TupleFilter child) {
+        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+    }
+
+    @Override
+    public String toString() {
+        return "ConstantFilter [constant=" + constantValues + "]";
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        return constantValues.size() > 0;
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return true;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        return this.constantValues;
+    }
+
+    @Override
+    public byte[] serialize() {
+        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+        int size = this.constantValues.size();
+        BytesUtil.writeVInt(size, buffer);
+        for (String val : this.constantValues) {
+            BytesUtil.writeUTFString(val, buffer);
+        }
+        byte[] result = new byte[buffer.position()];
+        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+        return result;
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+        this.constantValues.clear();
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        int size = BytesUtil.readVInt(buffer);
+        for (int i = 0; i < size; i++) {
+            this.constantValues.add(BytesUtil.readUTFString(buffer));
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
new file mode 100644
index 0000000..3c2d395
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/DynamicTupleFilter.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public class DynamicTupleFilter extends TupleFilter {
+
+    private String variableName;
+
+    public DynamicTupleFilter(String name) {
+        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.DYNAMIC);
+        this.variableName = name;
+    }
+
+    public String getVariableName() {
+        return variableName;
+    }
+
+    @Override
+    public void addChild(TupleFilter child) {
+        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
+    }
+
+    @Override
+    public String toString() {
+        return "DynamicFilter [variableName=" + variableName + "]";
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        return true;
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return true;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public byte[] serialize() {
+        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+        BytesUtil.writeUTFString(variableName, buffer);
+        byte[] result = new byte[buffer.position()];
+        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+        return result;
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        this.variableName = BytesUtil.readUTFString(buffer);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
new file mode 100644
index 0000000..9cfe08d
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
@@ -0,0 +1,107 @@
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public class ExtractTupleFilter extends TupleFilter {
+
+    private int date;
+    private List<String> values;
+
+    public ExtractTupleFilter(FilterOperatorEnum op) {
+        super(new ArrayList<TupleFilter>(3), op);
+        assert (op == FilterOperatorEnum.EXTRACT);
+        this.values = new ArrayList<String>(1);
+        this.values.add(null);
+        this.date = 0;
+    }
+
+    @Override
+    public String toString() {
+        return "ExtractTupleFilter=[children=" + this.children + "]";
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return false;
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        // extract tuple value
+        String extractType = null;
+        String tupleValue = null;
+        for (TupleFilter filter : this.children) {
+            filter.evaluate(tuple);
+            if (filter instanceof ConstantTupleFilter) {
+                tupleValue = filter.getValues().iterator().next();
+            } else if (filter instanceof CompareTupleFilter) {
+                extractType = filter.getValues().iterator().next();
+            }
+        }
+
+        // extract date
+        this.date = extractDate(extractType, Integer.valueOf(tupleValue));
+        return true;
+    }
+
+    private int extractDate(String type, int inDate) {
+        // this shifts the epoch back to astronomical year -4800 instead of the
+        // start of the Christian era in year AD 1 of the proleptic Gregorian
+        // calendar.
+        int j = inDate + 32044;
+        int g = j / 146097;
+        int dg = j % 146097;
+        int c = (dg / 36524 + 1) * 3 / 4;
+        int dc = dg - c * 36524;
+        int b = dc / 1461;
+        int db = dc % 1461;
+        int a = (db / 365 + 1) * 3 / 4;
+        int da = db - a * 365;
+
+        // integer number of full years elapsed since March 1, 4801 BC
+        int y = g * 400 + c * 100 + b * 4 + a;
+        // integer number of full months elapsed since the last March 1
+        int m = (da * 5 + 308) / 153 - 2;
+        // number of days elapsed since day 1 of the month
+        int d = da - (m + 4) * 153 / 5 + 122;
+        int year = y - 4800 + (m + 2) / 12;
+        int month = (m + 2) % 12 + 1;
+        int day = d + 1;
+        if ("YEAR".equalsIgnoreCase(type)) {
+            return year;
+        }
+        if ("MONTH".equalsIgnoreCase(type)) {
+            return month;
+        }
+        if ("DAY".equalsIgnoreCase(type)) {
+            return day;
+        }
+        return -1;
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        this.values.set(0, String.valueOf(this.date));
+        return this.values;
+    }
+
+    @Override
+    public byte[] serialize() {
+        return new byte[0];
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
new file mode 100644
index 0000000..c868737
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/LogicalTupleFilter.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import com.kylinolap.metadata.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+
+public class LogicalTupleFilter extends TupleFilter {
+
+    public LogicalTupleFilter(FilterOperatorEnum op) {
+        super(new ArrayList<TupleFilter>(2), op);
+        boolean opGood = (op == FilterOperatorEnum.AND || op == FilterOperatorEnum.OR || op == FilterOperatorEnum.NOT);
+        if (opGood == false)
+            throw new IllegalArgumentException("Unsupported operator " + op);
+    }
+
+    private LogicalTupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
+        super(filters, op);
+    }
+
+    @Override
+    public TupleFilter copy() {
+        List<TupleFilter> cloneChildren = new LinkedList<TupleFilter>(children);
+        TupleFilter cloneTuple = new LogicalTupleFilter(cloneChildren, operator);
+        return cloneTuple;
+    }
+
+    @Override
+    public TupleFilter reverse() {
+        switch (operator) {
+        case NOT:
+            assert (children.size() == 1);
+            return children.get(0);
+        case AND:
+        case OR:
+            LogicalTupleFilter reverse = new LogicalTupleFilter(REVERSE_OP_MAP.get(operator));
+            for (TupleFilter child : children) {
+                reverse.addChild(child.reverse());
+            }
+            return reverse;
+        default:
+            throw new IllegalStateException();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "LogicalFilter [operator=" + operator + ", children=" + children + "]";
+    }
+
+    @Override
+    public boolean evaluate(ITuple tuple) {
+        switch (this.operator) {
+        case AND:
+            return evalAnd(tuple);
+        case OR:
+            return evalOr(tuple);
+        case NOT:
+            return evalNot(tuple);
+        default:
+            return false;
+        }
+    }
+
+    private boolean evalAnd(ITuple tuple) {
+        for (TupleFilter filter : this.children) {
+            if (!filter.evaluate(tuple)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean evalOr(ITuple tuple) {
+        for (TupleFilter filter : this.children) {
+            if (filter.evaluate(tuple)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean evalNot(ITuple tuple) {
+        return !this.children.get(0).evaluate(tuple);
+    }
+
+    @Override
+    public Collection<String> getValues() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return true;
+    }
+
+    @Override
+    public byte[] serialize() {
+        return new byte[0];
+    }
+
+    @Override
+    public void deserialize(byte[] bytes) {
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
new file mode 100644
index 0000000..fa5e624
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.filter;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import com.kylinolap.metadata.tuple.ITuple;
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public abstract class TupleFilter {
+
+    public enum FilterOperatorEnum {
+        EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34);
+
+        private final int value;
+
+        private FilterOperatorEnum(int v) {
+            this.value = v;
+        }
+
+        public int getValue() {
+            return this.value;
+        }
+    }
+
+    public static final int BUFFER_SIZE = 10240;
+
+    protected static final Map<FilterOperatorEnum, FilterOperatorEnum> REVERSE_OP_MAP = Maps.newHashMap();
+    protected static final Map<FilterOperatorEnum, FilterOperatorEnum> SWAP_OP_MAP = Maps.newHashMap();
+
+    static {
+        REVERSE_OP_MAP.put(FilterOperatorEnum.EQ, FilterOperatorEnum.NEQ);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.NEQ, FilterOperatorEnum.EQ);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.GT, FilterOperatorEnum.LTE);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.LTE, FilterOperatorEnum.GT);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.LT, FilterOperatorEnum.GTE);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.GTE, FilterOperatorEnum.LT);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.IN, FilterOperatorEnum.NOTIN);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.NOTIN, FilterOperatorEnum.IN);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.ISNULL, FilterOperatorEnum.ISNOTNULL);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.ISNOTNULL, FilterOperatorEnum.ISNULL);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.AND, FilterOperatorEnum.OR);
+        REVERSE_OP_MAP.put(FilterOperatorEnum.OR, FilterOperatorEnum.AND);
+
+        SWAP_OP_MAP.put(FilterOperatorEnum.EQ, FilterOperatorEnum.EQ);
+        SWAP_OP_MAP.put(FilterOperatorEnum.NEQ, FilterOperatorEnum.NEQ);
+        SWAP_OP_MAP.put(FilterOperatorEnum.GT, FilterOperatorEnum.LT);
+        SWAP_OP_MAP.put(FilterOperatorEnum.LTE, FilterOperatorEnum.GTE);
+        SWAP_OP_MAP.put(FilterOperatorEnum.LT, FilterOperatorEnum.GT);
+        SWAP_OP_MAP.put(FilterOperatorEnum.GTE, FilterOperatorEnum.LTE);
+    }
+
+    protected final List<TupleFilter> children;
+    protected FilterOperatorEnum operator;
+    protected boolean hasChildren;
+
+    protected TupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
+        this.children = filters;
+        this.operator = op;
+    }
+
+    public void addChild(TupleFilter child) {
+        children.add(child);
+    }
+
+    final public void addChildren(List<? extends TupleFilter> children) {
+        for (TupleFilter c : children)
+            addChild(c); // subclass overrides addChild()
+    }
+
+    public List<? extends TupleFilter> getChildren() {
+        return children;
+    }
+
+    public boolean hasChildren() {
+        return children != null && !children.isEmpty();
+    }
+
+    public FilterOperatorEnum getOperator() {
+        return operator;
+    }
+
+    public TupleFilter copy() {
+        throw new UnsupportedOperationException();
+    }
+
+    public TupleFilter reverse() {
+        throw new UnsupportedOperationException();
+    }
+
+    public TupleFilter flatFilter() {
+        return flattenInternal(this);
+    }
+
+    private TupleFilter flattenInternal(TupleFilter filter) {
+        TupleFilter flatFilter = null;
+        if (!(filter instanceof LogicalTupleFilter)) {
+            flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+            flatFilter.addChild(filter);
+            return flatFilter;
+        }
+
+        // post-order recursive travel
+        FilterOperatorEnum op = filter.getOperator();
+        List<TupleFilter> andChildren = new LinkedList<TupleFilter>();
+        List<TupleFilter> orChildren = new LinkedList<TupleFilter>();
+        for (TupleFilter child : filter.getChildren()) {
+            TupleFilter flatChild = flattenInternal(child);
+            FilterOperatorEnum childOp = flatChild.getOperator();
+            if (childOp == FilterOperatorEnum.AND) {
+                andChildren.add(flatChild);
+            } else if (childOp == FilterOperatorEnum.OR) {
+                orChildren.add(flatChild);
+            } else {
+                throw new IllegalStateException("Filter is " + filter + " and child is " + flatChild);
+            }
+        }
+
+        // boolean algebra flatten
+        if (op == FilterOperatorEnum.AND) {
+            flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+            for (TupleFilter andChild : andChildren) {
+                flatFilter.addChildren(andChild.getChildren());
+            }
+            if (!orChildren.isEmpty()) {
+                List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter);
+                flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+                flatFilter.addChildren(fullAndFilters);
+            }
+        } else if (op == FilterOperatorEnum.OR) {
+            flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+            for (TupleFilter orChild : orChildren) {
+                flatFilter.addChildren(orChild.getChildren());
+            }
+            flatFilter.addChildren(andChildren);
+        } else if (op == FilterOperatorEnum.NOT) {
+            assert (filter.children.size() == 1);
+            TupleFilter reverse = filter.children.get(0).reverse();
+            flatFilter = flattenInternal(reverse);
+        } else {
+            throw new IllegalStateException("Filter is " + filter);
+        }
+        return flatFilter;
+    }
+
+    private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter) {
+        List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>();
+        oldProductFilters.add(partialAndFilter);
+        for (TupleFilter orFilter : leftOrFilters) {
+            List<TupleFilter> newProductFilters = new LinkedList<TupleFilter>();
+            for (TupleFilter orChildFilter : orFilter.getChildren()) {
+                for (TupleFilter productFilter : oldProductFilters) {
+                    TupleFilter fullAndFilter = productFilter.copy();
+                    fullAndFilter.addChildren(orChildFilter.getChildren());
+                    newProductFilters.add(fullAndFilter);
+                }
+            }
+            oldProductFilters = newProductFilters;
+        }
+        return oldProductFilters;
+    }
+
+    public abstract boolean isEvaluable();
+
+    public abstract boolean evaluate(ITuple tuple);
+
+    public abstract Collection<String> getValues();
+
+    public abstract byte[] serialize();
+
+    public abstract void deserialize(byte[] bytes);
+
+    public static boolean isEvaluableRecursively(TupleFilter filter) {
+        if (filter == null)
+            return true;
+
+        if (filter.isEvaluable() == false)
+            return false;
+
+        for (TupleFilter child : filter.getChildren()) {
+            if (isEvaluableRecursively(child) == false)
+                return false;
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
new file mode 100644
index 0000000..2ca9356
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilterSerializer.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+
+import com.kylinolap.common.util.BytesUtil;
+
+/**
+ * http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
+ * 
+ * @author xjiang
+ * 
+ */
+public class TupleFilterSerializer {
+
+    public static interface Decorator {
+        TupleFilter onSerialize(TupleFilter filter);
+    }
+
+    private static final int BUFFER_SIZE = 65536;
+    private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>();
+
+    static {
+        for (TupleFilter.FilterOperatorEnum op : TupleFilter.FilterOperatorEnum.values()) {
+            ID_OP_MAP.put(op.getValue(), op);
+        }
+    }
+
+    public static byte[] serialize(TupleFilter rootFilter) {
+        return serialize(rootFilter, null);
+    }
+
+    public static byte[] serialize(TupleFilter rootFilter, Decorator decorator) {
+        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+        internalSerialize(rootFilter, decorator, buffer);
+        byte[] result = new byte[buffer.position()];
+        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
+        return result;
+    }
+
+    private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+        if (decorator != null) { // give decorator a chance to manipulate the
+                                 // output filter
+            filter = decorator.onSerialize(filter);
+        }
+
+        if (filter == null) {
+            return;
+        }
+
+        if (filter.hasChildren()) {
+            // serialize filter+true
+            serializeFilter(1, filter, decorator, buffer);
+            // serialize children
+            for (TupleFilter child : filter.getChildren()) {
+                internalSerialize(child, decorator, buffer);
+            }
+            // serialize none
+            serializeFilter(-1, filter, decorator, buffer);
+        } else {
+            // serialize filter+false
+            serializeFilter(0, filter, decorator, buffer);
+        }
+    }
+
+    private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+        if (flag < 0) {
+            BytesUtil.writeVInt(-1, buffer);
+        } else {
+            byte[] bytes = filter.serialize();
+            int opVal = filter.getOperator().getValue();
+            BytesUtil.writeVInt(opVal, buffer);
+            BytesUtil.writeByteArray(bytes, buffer);
+            BytesUtil.writeVInt(flag, buffer);
+        }
+    }
+
+    public static TupleFilter deserialize(byte[] bytes) {
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        TupleFilter rootFilter = null;
+        Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
+        while (buffer.hasRemaining()) {
+            int opVal = BytesUtil.readVInt(buffer);
+            if (opVal < 0) {
+                parentStack.pop();
+                continue;
+            }
+
+            // deserialize filter
+            TupleFilter filter = createTupleFilter(opVal);
+            byte[] filetrBytes = BytesUtil.readByteArray(buffer);
+            filter.deserialize(filetrBytes);
+
+            if (rootFilter == null) {
+                // push root to stack
+                rootFilter = filter;
+                parentStack.push(filter);
+                BytesUtil.readVInt(buffer);
+                continue;
+            }
+
+            // add filter to parent
+            TupleFilter parentFilter = parentStack.peek();
+            if (parentFilter != null) {
+                parentFilter.addChild(filter);
+            }
+
+            // push filter to stack or not based on having children or not
+            int hasChild = BytesUtil.readVInt(buffer);
+            if (hasChild == 1) {
+                parentStack.push(filter);
+            }
+        }
+        return rootFilter;
+    }
+
+    private static TupleFilter createTupleFilter(int opVal) {
+        TupleFilter.FilterOperatorEnum op = ID_OP_MAP.get(opVal);
+        if (op == null) {
+            throw new IllegalStateException("operator value is " + opVal);
+        }
+        TupleFilter filter = null;
+        switch (op) {
+        case AND:
+        case OR:
+        case NOT:
+            filter = new LogicalTupleFilter(op);
+            break;
+        case EQ:
+        case NEQ:
+        case LT:
+        case LTE:
+        case GT:
+        case GTE:
+        case IN:
+        case ISNULL:
+        case ISNOTNULL:
+            filter = new CompareTupleFilter(op);
+            break;
+        case EXTRACT:
+            filter = new ExtractTupleFilter(op);
+            break;
+        case CASE:
+            filter = new CaseTupleFilter();
+            break;
+        case COLUMN:
+            filter = new ColumnTupleFilter(null);
+            break;
+        case CONSTANT:
+            filter = new ConstantTupleFilter();
+            break;
+        case DYNAMIC:
+            filter = new DynamicTupleFilter(null);
+            break;
+        default:
+            throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+        }
+
+        return filter;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java b/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
index 57651d0..7169e6b 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/realization/IRealization.java
@@ -35,8 +35,6 @@ public interface IRealization {
 
     public List<MeasureDesc> getMeasures();
 
-    public List<TblColRef> getDimensions();
-
     public boolean isReady();
 
     public String getName();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java b/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
index 5922519..f7b3434 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/realization/SQLDigest.java
@@ -2,10 +2,10 @@ package com.kylinolap.metadata.realization;
 
 import java.util.Collection;
 
+import com.kylinolap.metadata.filter.TupleFilter;
 import com.kylinolap.metadata.model.FunctionDesc;
 import com.kylinolap.metadata.model.JoinDesc;
 import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.filter.TupleFilter;
 
 /**
  * Created by Hongbin Ma(Binmahone) on 1/8/15.

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
new file mode 100644
index 0000000..e5a7c39
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITuple.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.tuple;
+
+import java.util.List;
+
+import com.kylinolap.metadata.model.TblColRef;
+
+/**
+ * Tuple is a record row, contains multiple values being lookup by either field
+ * (optiq notion) or column (kylin notion).
+ * 
+ * @author yangli9
+ */
+public interface ITuple {
+
+    public List<String> getAllFields();
+
+    public List<TblColRef> getAllColumns();
+
+    public Object[] getAllValues();
+
+    public Object getValue(TblColRef col);
+
+    public Object getValue(String field);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
new file mode 100644
index 0000000..862649d
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/tuple/ITupleIterator.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.metadata.tuple;
+
+/**
+ * @author xjiang
+ */
+public interface ITupleIterator {
+    public static final ITupleIterator EMPTY_TUPLE_ITERATOR = new ITupleIterator() {
+        @Override
+        public boolean hasNext() {
+            return false;
+        }
+
+        @Override
+        public ITuple next() {
+            return null;
+        }
+
+        @Override
+        public void close() {
+        }
+    };
+
+    public boolean hasNext();
+
+    public ITuple next();
+
+    public void close();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java b/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
index d3adca5..82f2101 100644
--- a/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/com/kylinolap/query/enumerator/CubeEnumerator.java
@@ -30,10 +30,10 @@ import org.slf4j.LoggerFactory;
 import com.kylinolap.query.relnode.OLAPContext;
 import com.kylinolap.storage.IStorageEngine;
 import com.kylinolap.storage.StorageEngineFactory;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
-import com.kylinolap.storage.tuple.ITuple;
-import com.kylinolap.storage.tuple.ITupleIterator;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
+import com.kylinolap.metadata.tuple.ITuple;
+import com.kylinolap.metadata.tuple.ITupleIterator;
 
 /**
  * @author xjiang

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
index f8ec2d9..3ea622a 100644
--- a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
+++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java
@@ -33,7 +33,7 @@ import com.kylinolap.metadata.model.TblColRef;
 import com.kylinolap.metadata.realization.IRealization;
 import com.kylinolap.query.schema.OLAPSchema;
 import com.kylinolap.storage.StorageContext;
-import com.kylinolap.storage.filter.TupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
 
 /**
  * @author xjiang
@@ -103,7 +103,6 @@ public class OLAPContext {
     // cube metadata
     public IRealization realization;
 
-
     public Collection<TblColRef> allColumns = new HashSet<TblColRef>();
     public Collection<TblColRef> groupByColumns = new ArrayList<TblColRef>();
     public Collection<TblColRef> metricsColumns = new HashSet<TblColRef>();
@@ -126,7 +125,7 @@ public class OLAPContext {
 
     public SQLDigest getSQLDigest() {
         if (sqlDigest == null)
-            sqlDigest = new SQLDigest(firstTableScan.getTableName(), joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations);
+            sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations);
         return sqlDigest;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
index 66265c2..550d154 100644
--- a/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPFilterRel.java
@@ -50,15 +50,15 @@ import org.eigenbase.util.NlsString;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.filter.CaseTupleFilter;
-import com.kylinolap.storage.filter.ColumnTupleFilter;
-import com.kylinolap.storage.filter.CompareTupleFilter;
-import com.kylinolap.storage.filter.ConstantTupleFilter;
-import com.kylinolap.storage.filter.DynamicTupleFilter;
-import com.kylinolap.storage.filter.ExtractTupleFilter;
-import com.kylinolap.storage.filter.LogicalTupleFilter;
-import com.kylinolap.storage.filter.TupleFilter;
-import com.kylinolap.storage.filter.TupleFilter.FilterOperatorEnum;
+import com.kylinolap.metadata.filter.CaseTupleFilter;
+import com.kylinolap.metadata.filter.ColumnTupleFilter;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.ConstantTupleFilter;
+import com.kylinolap.metadata.filter.DynamicTupleFilter;
+import com.kylinolap.metadata.filter.ExtractTupleFilter;
+import com.kylinolap.metadata.filter.LogicalTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter.FilterOperatorEnum;
 
 /**
  * @author xjiang

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java b/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
index a131752..2352831 100644
--- a/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
+++ b/storage/src/main/java/com/kylinolap/storage/IStorageEngine.java
@@ -16,7 +16,7 @@
 package com.kylinolap.storage;
 
 import com.kylinolap.metadata.realization.SQLDigest;
-import com.kylinolap.storage.tuple.ITupleIterator;
+import com.kylinolap.metadata.tuple.ITupleIterator;
 
 /**
  * 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
index 22fc57f..8f249e6 100644
--- a/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
+++ b/storage/src/main/java/com/kylinolap/storage/filter/BitMapFilterEvaluator.java
@@ -3,6 +3,9 @@ package com.kylinolap.storage.filter;
 import java.util.List;
 
 import com.kylinolap.dict.Dictionary;
+import com.kylinolap.metadata.filter.CompareTupleFilter;
+import com.kylinolap.metadata.filter.LogicalTupleFilter;
+import com.kylinolap.metadata.filter.TupleFilter;
 import com.kylinolap.metadata.model.TblColRef;
 
 import it.uniroma3.mat.extendedset.intset.ConciseSet;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
deleted file mode 100644
index 3629742..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/CaseTupleFilter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.storage.filter;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * @author xjiang
- * 
- */
-public class CaseTupleFilter extends TupleFilter {
-
-    private List<TupleFilter> whenFilters;
-    private List<TupleFilter> thenFilters;
-    private TupleFilter elseFilter;
-    private Collection<String> values;
-    private int filterIndex;
-
-    public CaseTupleFilter() {
-        super(new ArrayList<TupleFilter>(), FilterOperatorEnum.CASE);
-        this.filterIndex = 0;
-        this.values = Collections.emptyList();
-        this.whenFilters = new ArrayList<TupleFilter>();
-        this.thenFilters = new ArrayList<TupleFilter>();
-        this.elseFilter = null;
-    }
-
-    @Override
-    public void addChild(TupleFilter child) {
-        super.addChild(child);
-        if (this.filterIndex % 2 == 0) {
-            this.whenFilters.add(child);
-        } else {
-            this.thenFilters.add(child);
-        }
-        this.filterIndex++;
-    }
-
-    @Override
-    public String toString() {
-        return "CaseTupleFilter [when=" + whenFilters + ", then=" + thenFilters + ", else=" + elseFilter + ", children=" + children + "]";
-    }
-
-    @Override
-    public boolean evaluate(ITuple tuple) {
-        if (whenFilters.size() != thenFilters.size()) {
-            elseFilter = whenFilters.remove(whenFilters.size() - 1);
-        }
-        boolean matched = false;
-        for (int i = 0; i < whenFilters.size(); i++) {
-            TupleFilter whenFilter = whenFilters.get(i);
-            if (whenFilter.evaluate(tuple)) {
-                TupleFilter thenFilter = thenFilters.get(i);
-                thenFilter.evaluate(tuple);
-                values = thenFilter.getValues();
-                matched = true;
-                break;
-            }
-        }
-        if (!matched) {
-            if (elseFilter != null) {
-                elseFilter.evaluate(tuple);
-                values = elseFilter.getValues();
-            } else {
-                values = Collections.emptyList();
-            }
-        }
-
-        return true;
-    }
-
-    @Override
-    public boolean isEvaluable() {
-        return false;
-    }
-
-    @Override
-    public Collection<String> getValues() {
-        return this.values;
-    }
-
-    @Override
-    public byte[] serialize() {
-        return new byte[0];
-    }
-
-    @Override
-    public void deserialize(byte[] bytes) {
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
deleted file mode 100644
index 746cc96..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/ColumnTupleFilter.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.metadata.model.ColumnDesc;
-import com.kylinolap.metadata.model.TableDesc;
-import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * 
- * @author xjiang
- * 
- */
-public class ColumnTupleFilter extends TupleFilter {
-
-    private TblColRef columnRef;
-    private Object tupleValue;
-    private List<String> values;
-
-    public ColumnTupleFilter(TblColRef column) {
-        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
-        this.columnRef = column;
-        this.values = new ArrayList<String>(1);
-        this.values.add(null);
-    }
-
-    public TblColRef getColumn() {
-        return columnRef;
-    }
-
-    public void setColumn(TblColRef col) {
-        this.columnRef = col;
-    }
-
-    @Override
-    public void addChild(TupleFilter child) {
-        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
-    }
-
-    @Override
-    public String toString() {
-        return "ColumnFilter [column=" + columnRef + "]";
-    }
-
-    @Override
-    public boolean evaluate(ITuple tuple) {
-        this.tupleValue = tuple.getValue(columnRef);
-        return true;
-    }
-
-    @Override
-    public boolean isEvaluable() {
-        return true;
-    }
-
-    @Override
-    public Collection<String> getValues() {
-        this.values.set(0, (String) this.tupleValue);
-        return this.values;
-    }
-
-    @Override
-    public byte[] serialize() {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-        String table = columnRef.getTable();
-        BytesUtil.writeUTFString(table, buffer);
-
-        String columnName = columnRef.getName();
-        BytesUtil.writeUTFString(columnName, buffer);
-
-        String dataType = columnRef.getDatatype();
-        BytesUtil.writeUTFString(dataType, buffer);
-
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
-    }
-
-    @Override
-    public void deserialize(byte[] bytes) {
-        ColumnDesc column = new ColumnDesc();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
-        String tableName = BytesUtil.readUTFString(buffer);
-        if (tableName != null) {
-            TableDesc table = new TableDesc();
-            table.setName(tableName);
-            column.setTable(table);
-        }
-
-        column.setName(BytesUtil.readUTFString(buffer));
-        column.setDatatype(BytesUtil.readUTFString(buffer));
-
-        this.columnRef = new TblColRef(column);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
deleted file mode 100644
index c7cc0c2..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/CompareTupleFilter.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * @author xjiang
- */
-public class CompareTupleFilter extends TupleFilter {
-
-    private TblColRef column;
-    private Collection<String> conditionValues;
-    private String firstCondValue;
-    private Map<String, String> dynamicVariables;
-    private String nullString;
-
-    public CompareTupleFilter(FilterOperatorEnum op) {
-        super(new ArrayList<TupleFilter>(2), op);
-        this.conditionValues = new HashSet<String>();
-        this.dynamicVariables = new HashMap<String, String>();
-        boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
-                || op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
-                || op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
-                || op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN //
-                || op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL);
-        if (opGood == false)
-            throw new IllegalArgumentException("Unsupported operator " + op);
-    }
-
-    private CompareTupleFilter(CompareTupleFilter another) {
-        super(new ArrayList<TupleFilter>(another.children), another.operator);
-        this.column = another.column;
-        this.conditionValues = new HashSet<String>();
-        this.conditionValues.addAll(another.conditionValues);
-        this.dynamicVariables = new HashMap<String, String>();
-        this.dynamicVariables.putAll(another.dynamicVariables);
-    }
-
-    @Override
-    public void addChild(TupleFilter child) {
-        super.addChild(child);
-        if (child instanceof ColumnTupleFilter) {
-            ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
-            if (this.column != null) {
-                throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
-            }
-            this.column = columnFilter.getColumn();
-            // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
-            if (!this.conditionValues.isEmpty() && needSwapOperator()) {
-                this.operator = SWAP_OP_MAP.get(this.operator);
-            }
-        } else if (child instanceof ConstantTupleFilter) {
-            this.conditionValues.addAll(child.getValues());
-            this.firstCondValue = this.conditionValues.iterator().next();
-        } else if (child instanceof DynamicTupleFilter) {
-            DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
-            this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
-        }
-        //TODO
-        //        else if (child instanceof ExtractTupleFilter) {
-        //        } else if (child instanceof CaseTupleFilter) {
-        //        }
-    }
-
-    private boolean needSwapOperator() {
-        return operator == FilterOperatorEnum.LT || operator == FilterOperatorEnum.GT || operator == FilterOperatorEnum.LTE || operator == FilterOperatorEnum.GTE;
-    }
-
-    @Override
-    public Collection<String> getValues() {
-        return conditionValues;
-    }
-
-    public String getFirstValue() {
-        return firstCondValue;
-    }
-
-    public TblColRef getColumn() {
-        return column;
-    }
-
-    public Map<String, String> getVariables() {
-        return dynamicVariables;
-    }
-
-    public void bindVariable(String variable, String value) {
-        this.dynamicVariables.put(variable, value);
-        this.conditionValues.add(value);
-        this.firstCondValue = this.conditionValues.iterator().next();
-    }
-
-    public String getNullString() {
-        return nullString;
-    }
-
-    public void setNullString(String nullString) {
-        this.nullString = nullString;
-    }
-
-    @Override
-    public TupleFilter copy() {
-        return new CompareTupleFilter(this);
-    }
-
-    @Override
-    public TupleFilter reverse() {
-        TupleFilter reverse = copy();
-        reverse.operator = REVERSE_OP_MAP.get(this.operator);
-        return reverse;
-    }
-
-    @Override
-    public String toString() {
-        return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]";
-    }
-
-    // TODO requires generalize, currently only evaluates COLUMN {op} CONST
-    @Override
-    public boolean evaluate(ITuple tuple) {
-        // extract tuple value
-        String tupleValue = null;
-        for (TupleFilter filter : this.children) {
-            if (isConstant(filter) == false) {
-                filter.evaluate(tuple);
-                tupleValue = filter.getValues().iterator().next();
-            }
-        }
-
-        // consider null string
-        if (nullString != null && nullString.equals(tupleValue)) {
-            tupleValue = null;
-        }
-        if (tupleValue == null) {
-            if (operator == FilterOperatorEnum.ISNULL)
-                return true;
-            else
-                return false;
-        }
-
-        // always false if compare to null
-        if (firstCondValue.equals(nullString))
-            return false;
-
-        // tricky here -- order is ensured by string compare (even for number columns)
-        // because it's row key ID (not real value) being compared
-        int comp = tupleValue.compareTo(firstCondValue);
-
-        boolean result;
-        switch (operator) {
-        case EQ:
-            result = comp == 0;
-            break;
-        case NEQ:
-            result = comp != 0;
-            break;
-        case LT:
-            result = comp < 0;
-            break;
-        case LTE:
-            result = comp <= 0;
-            break;
-        case GT:
-            result = comp > 0;
-            break;
-        case GTE:
-            result = comp >= 0;
-            break;
-        case IN:
-            result = conditionValues.contains(tupleValue);
-            break;
-        case NOTIN:
-            result = !conditionValues.contains(tupleValue);
-            break;
-        default:
-            result = false;
-        }
-        return result;
-    }
-
-    private boolean isConstant(TupleFilter filter) {
-        return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
-    }
-
-    @Override
-    public boolean isEvaluable() {
-        return column != null && !conditionValues.isEmpty();
-    }
-
-    @Override
-    public byte[] serialize() {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-        int size = this.dynamicVariables.size();
-        BytesUtil.writeVInt(size, buffer);
-        for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
-            BytesUtil.writeUTFString(entry.getKey(), buffer);
-            BytesUtil.writeUTFString(entry.getValue(), buffer);
-        }
-        BytesUtil.writeAsciiString(nullString, buffer);
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
-    }
-
-    @Override
-    public void deserialize(byte[] bytes) {
-        this.dynamicVariables.clear();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
-        int size = BytesUtil.readVInt(buffer);
-        for (int i = 0; i < size; i++) {
-            String nameString = BytesUtil.readUTFString(buffer);
-            String valueString = BytesUtil.readUTFString(buffer);
-            bindVariable(nameString, valueString);
-        }
-        this.nullString = BytesUtil.readAsciiString(buffer);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
deleted file mode 100644
index 9457d03..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/ConstantTupleFilter.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * 
- * @author xjiang
- * 
- */
-public class ConstantTupleFilter extends TupleFilter {
-
-    public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
-    public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
-
-    private Collection<String> constantValues;
-
-    public ConstantTupleFilter() {
-        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
-        this.constantValues = new HashSet<String>();
-    }
-
-    public ConstantTupleFilter(String value) {
-        this();
-        this.constantValues.add(value);
-    }
-
-    public ConstantTupleFilter(Collection<String> values) {
-        this();
-        this.constantValues.addAll(values);
-    }
-
-    @Override
-    public void addChild(TupleFilter child) {
-        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
-    }
-
-    @Override
-    public String toString() {
-        return "ConstantFilter [constant=" + constantValues + "]";
-    }
-
-    @Override
-    public boolean evaluate(ITuple tuple) {
-        return constantValues.size() > 0;
-    }
-
-    @Override
-    public boolean isEvaluable() {
-        return true;
-    }
-
-    @Override
-    public Collection<String> getValues() {
-        return this.constantValues;
-    }
-
-    @Override
-    public byte[] serialize() {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-        int size = this.constantValues.size();
-        BytesUtil.writeVInt(size, buffer);
-        for (String val : this.constantValues) {
-            BytesUtil.writeUTFString(val, buffer);
-        }
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
-    }
-
-    @Override
-    public void deserialize(byte[] bytes) {
-        this.constantValues.clear();
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
-        int size = BytesUtil.readVInt(buffer);
-        for (int i = 0; i < size; i++) {
-            this.constantValues.add(BytesUtil.readUTFString(buffer));
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/94352aec/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java b/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
deleted file mode 100644
index b2f85d5..0000000
--- a/storage/src/main/java/com/kylinolap/storage/filter/DynamicTupleFilter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.storage.filter;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.storage.tuple.ITuple;
-
-/**
- * 
- * @author xjiang
- * 
- */
-public class DynamicTupleFilter extends TupleFilter {
-
-    private String variableName;
-
-    public DynamicTupleFilter(String name) {
-        super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.DYNAMIC);
-        this.variableName = name;
-    }
-
-    public String getVariableName() {
-        return variableName;
-    }
-
-    @Override
-    public void addChild(TupleFilter child) {
-        throw new UnsupportedOperationException("This is " + this + " and child is " + child);
-    }
-
-    @Override
-    public String toString() {
-        return "DynamicFilter [variableName=" + variableName + "]";
-    }
-
-    @Override
-    public boolean evaluate(ITuple tuple) {
-        return true;
-    }
-
-    @Override
-    public boolean isEvaluable() {
-        return true;
-    }
-
-    @Override
-    public Collection<String> getValues() {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public byte[] serialize() {
-        ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-        BytesUtil.writeUTFString(variableName, buffer);
-        byte[] result = new byte[buffer.position()];
-        System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
-        return result;
-    }
-
-    @Override
-    public void deserialize(byte[] bytes) {
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
-        this.variableName = BytesUtil.readUTFString(buffer);
-    }
-
-}


Mime
View raw message