Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D4BC2182E8 for ; Fri, 26 Feb 2016 09:55:28 +0000 (UTC) Received: (qmail 28742 invoked by uid 500); 26 Feb 2016 09:55:19 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 28705 invoked by uid 500); 26 Feb 2016 09:55:19 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 28694 invoked by uid 99); 26 Feb 2016 09:55:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Feb 2016 09:55:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CF0FBE00A1; Fri, 26 Feb 2016 09:55:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mahongbin@apache.org To: commits@kylin.apache.org Date: Fri, 26 Feb 2016 09:55:18 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/4] kylin git commit: KYLIN-1453 cuboid sharding based on specific column Repository: kylin Updated Branches: refs/heads/2.x-staging 2e1d2f6b6 -> 294fc7078 KYLIN-1453 cuboid sharding based on specific column Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294fc707 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294fc707 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294fc707 Branch: refs/heads/2.x-staging Commit: 294fc70785ab009e6b4d8a12cbeb609d46f89a93 Parents: 3f5074e Author: honma Authored: Thu Feb 25 19:02:28 2016 +0800 Committer: honma Committed: Fri Feb 26 17:54:37 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BytesUtil.java | 18 +- .../dict/TupleFilterFunctionTransformer.java | 170 +++++++++++++++++++ .../dict/TupleFilterFunctionTranslator.java | 166 ------------------ .../kylin/metadata/filter/CaseTupleFilter.java | 7 +- .../metadata/filter/ColumnTupleFilter.java | 11 +- .../metadata/filter/CompareTupleFilter.java | 10 +- .../metadata/filter/ConstantTupleFilter.java | 16 +- .../metadata/filter/DynamicTupleFilter.java | 9 +- .../metadata/filter/ExtractTupleFilter.java | 7 +- .../metadata/filter/FunctionTupleFilter.java | 16 +- .../filter/ITupleFilterTransformer.java | 23 +++ .../metadata/filter/ITupleFilterTranslator.java | 26 --- .../metadata/filter/LogicalTupleFilter.java | 10 +- .../kylin/metadata/filter/TupleFilter.java | 5 +- .../metadata/filter/TupleFilterSerializer.java | 25 ++- .../common/coprocessor/FilterDecorator.java | 8 +- .../hbase/cube/v2/CubeSegmentScanner.java | 10 +- .../common/coprocessor/FilterBaseTest.java | 46 ++++- .../common/coprocessor/FilterEvaluateTest.java | 4 +- .../common/coprocessor/FilterSerializeTest.java | 26 ++- 20 files changed, 325 insertions(+), 288 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java index e01ce4f..4e0701c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java @@ -57,7 +57,6 @@ public class BytesUtil { return integer; } - public static long readLong(ByteBuffer buffer, int size) { long integer = 0; for (int i = 0; i < size; i++) { @@ -133,11 +132,15 @@ public class BytesUtil { // from WritableUtils // ============================================================================ + public static void writeVInt(int i, ByteBuffer out) { + writeVLong(i, out); + } public static void writeVLong(long i, ByteBuffer out) { + if (i >= -112 && i <= 127) { out.put((byte) i); return; @@ -203,6 +206,8 @@ public class BytesUtil { } public static void writeUnsigned(int num, int size, ByteBuffer out) { + + int mask = 0xff << ((size - 1) * 8); for (int i = size; i > 0; i--) { int v = (num & mask) >> (i - 1) * 8; @@ -222,6 +227,7 @@ public class BytesUtil { } public static void writeLong(long num, ByteBuffer out) { + for (int i = 0; i < 8; i++) { out.put((byte) num); num >>>= 8; @@ -257,6 +263,8 @@ public class BytesUtil { } int len = str.length(); BytesUtil.writeVInt(len, out); + + for (int i = 0; i < len; i++) { out.put((byte) str.charAt(i)); } @@ -335,7 +343,7 @@ public class BytesUtil { writeVInt(-1, out); return; } - writeVInt(array.length, out); + writeVInt(length, out); out.put(array, offset, length); } @@ -348,7 +356,7 @@ public class BytesUtil { in.get(array); return array; } - + public static int peekByteArrayLength(ByteBuffer in) { int start = in.position(); int arrayLen = readVInt(in); @@ -369,6 +377,7 @@ public class BytesUtil { writeVInt(array.length, out); byte b_true = (byte) 1; byte b_false = (byte) 0; + for (int i = 0; i < array.length; i++) { if (array[i]) out.put(b_true); @@ -428,7 +437,4 @@ public class BytesUtil { return sb.toString(); } - public static void main(String[] args) throws Exception { - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java new file mode 100644 index 0000000..096a28d --- /dev/null +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dict; + +import java.util.Collection; +import java.util.ListIterator; + +import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.metadata.filter.ColumnTupleFilter; +import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.FunctionTupleFilter; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; +import org.apache.kylin.metadata.filter.LogicalTupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.google.common.primitives.Primitives; + +/** + * only take effect when the compare filter has function + */ +public class TupleFilterFunctionTransformer implements ITupleFilterTransformer { + public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTransformer.class); + + private IDictionaryAware dictionaryAware; + + public TupleFilterFunctionTransformer(IDictionaryAware dictionaryAware) { + this.dictionaryAware = dictionaryAware; + } + + @Override + public TupleFilter transform(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof FunctionTupleFilter) { + translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof LogicalTupleFilter) { + ListIterator childIterator = (ListIterator) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter transformed = transform(childIterator.next()); + if (transformed != null) + childIterator.set(transformed); + } + } + return translated == null ? tupleFilter : translated; + } + + private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } + + @SuppressWarnings("unchecked") + private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { + if (compTupleFilter.getFunction() == null) + return null; + + FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + Collection inValues = Lists.newArrayList(); + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + Object computedVal = functionTupleFilter.invokeFunction(dictVal); + Class clazz = Primitives.wrap(computedVal.getClass()); + Object targetVal = compTupleFilter.getFirstValue(); + if (Primitives.isWrapperType(clazz)) + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); + + int comp = ((Comparable) computedVal).compareTo(targetVal); + boolean compResult = false; + switch (compTupleFilter.getOperator()) { + case EQ: + compResult = comp == 0; + break; + case NEQ: + compResult = comp != 0; + break; + case LT: + compResult = comp < 0; + break; + case LTE: + compResult = comp <= 0; + break; + case GT: + compResult = comp > 0; + break; + case GTE: + compResult = comp >= 0; + break; + case IN: + compResult = compTupleFilter.getValues().contains(computedVal.toString()); + break; + case NOTIN: + compResult = !compTupleFilter.getValues().contains(computedVal.toString()); + break; + default: + break; + } + if (compResult) { + inValues.add(dictVal); + } + } + translated.addChild(new ConstantTupleFilter(inValues)); + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java deleted file mode 100644 index 1c96dd4..0000000 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import java.util.ListIterator; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.FunctionTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.primitives.Primitives; - -/** - * only take effect when the compare filter has function - */ -public class TupleFilterFunctionTranslator implements ITupleFilterTranslator { - public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTranslator.class); - - private IDictionaryAware dictionaryAware; - - public TupleFilterFunctionTranslator(IDictionaryAware dictionaryAware) { - this.dictionaryAware = dictionaryAware; - } - - @Override - public TupleFilter translate(TupleFilter tupleFilter) { - TupleFilter translated = null; - if (tupleFilter instanceof CompareTupleFilter) { - translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof FunctionTupleFilter) { - translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof LogicalTupleFilter) { - ListIterator childIterator = (ListIterator) tupleFilter.getChildren().listIterator(); - while (childIterator.hasNext()) { - TupleFilter tempTranslated = translate(childIterator.next()); - if (tempTranslated != null) - childIterator.set(tempTranslated); - } - } - return translated == null ? tupleFilter : translated; - } - - private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } - - @SuppressWarnings("unchecked") - private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { - if (compTupleFilter.getFunction() == null) - return null; - - FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - Object computedVal = functionTupleFilter.invokeFunction(dictVal); - Class clazz = Primitives.wrap(computedVal.getClass()); - Object targetVal = compTupleFilter.getFirstValue(); - if (Primitives.isWrapperType(clazz)) - targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); - - int comp = ((Comparable) computedVal).compareTo(targetVal); - boolean compResult = false; - switch (compTupleFilter.getOperator()) { - case EQ: - compResult = comp == 0; - break; - case NEQ: - compResult = comp != 0; - break; - case LT: - compResult = comp < 0; - break; - case LTE: - compResult = comp <= 0; - break; - case GT: - compResult = comp > 0; - break; - case GTE: - compResult = comp >= 0; - break; - case IN: - compResult = compTupleFilter.getValues().contains(computedVal.toString()); - break; - case NOTIN: - compResult = !compTupleFilter.getValues().contains(computedVal.toString()); - break; - default: - break; - } - if (compResult) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java index 6dbc614..2b00d69 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.filter; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -101,12 +102,12 @@ public class CaseTupleFilter extends TupleFilter { } @Override - public byte[] serialize(IFilterCodeSystem cs) { - return new byte[0]; + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { + //serialize nothing } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java index 0d2a73d..029233d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java @@ -84,8 +84,7 @@ public class ColumnTupleFilter extends TupleFilter { } @Override - public byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + public void serialize(IFilterCodeSystem cs,ByteBuffer buffer) { String table = columnRef.getTable(); BytesUtil.writeUTFString(table, buffer); @@ -97,17 +96,13 @@ public class ColumnTupleFilter extends TupleFilter { String dataType = columnRef.getDatatype(); BytesUtil.writeUTFString(dataType, buffer); - - byte[] result = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); - return result; } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { + TableDesc table = null; ColumnDesc column = new ColumnDesc(); - ByteBuffer buffer = ByteBuffer.wrap(bytes); String tableName = BytesUtil.readUTFString(buffer); if (tableName != null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java index 248ab3b..fc0bab7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -214,23 +214,19 @@ public class CompareTupleFilter extends TupleFilter { @SuppressWarnings({ "unchecked", "rawtypes" }) @Override - public byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { int size = this.dynamicVariables.size(); BytesUtil.writeVInt(size, buffer); for (Map.Entry entry : this.dynamicVariables.entrySet()) { BytesUtil.writeUTFString(entry.getKey(), buffer); cs.serialize(entry.getValue(), buffer); } - byte[] result = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); - return result; } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { + this.dynamicVariables.clear(); - ByteBuffer buffer = ByteBuffer.wrap(bytes); int size = BytesUtil.readVInt(buffer); for (int i = 0; i < size; i++) { String name = BytesUtil.readUTFString(buffer); http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java index 3056a9c..db3eb4f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java @@ -21,8 +21,10 @@ package org.apache.kylin.metadata.filter; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; +import java.util.TreeSet; +import com.google.common.collect.Lists; +import org.apache.commons.collections.comparators.NullComparator; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; @@ -40,7 +42,7 @@ public class ConstantTupleFilter extends TupleFilter { public ConstantTupleFilter() { super(Collections. emptyList(), FilterOperatorEnum.CONSTANT); - this.constantValues = new HashSet(); + this.constantValues = Lists.newArrayList(); } public ConstantTupleFilter(Object value) { @@ -89,22 +91,18 @@ public class ConstantTupleFilter extends TupleFilter { @SuppressWarnings({ "unchecked", "rawtypes" }) @Override - public byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { int size = this.constantValues.size(); BytesUtil.writeVInt(size, buffer); for (Object val : this.constantValues) { cs.serialize(val, buffer); } - byte[] result = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); - return result; } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { + this.constantValues.clear(); - ByteBuffer buffer = ByteBuffer.wrap(bytes); int size = BytesUtil.readVInt(buffer); for (int i = 0; i < size; i++) { this.constantValues.add(cs.deserialize(buffer)); http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java index a482519..d9dc52a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java @@ -69,17 +69,12 @@ public class DynamicTupleFilter extends TupleFilter { } @Override - public byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { BytesUtil.writeUTFString(variableName, buffer); - byte[] result = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); - return result; } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.wrap(bytes); + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { this.variableName = BytesUtil.readUTFString(buffer); } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java index 6f7dfaf..591e64b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.filter; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -113,12 +114,12 @@ public class ExtractTupleFilter extends TupleFilter { } @Override - public byte[] serialize(IFilterCodeSystem cs) { - return new byte[0]; + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { + //do nothing } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java index 30bef97..2a08728 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java @@ -123,32 +123,20 @@ public class FunctionTupleFilter extends TupleFilter { } @Override - byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { BytesUtil.writeUTFString(name, buffer); BytesUtil.writeVInt(colPosition, buffer); BytesUtil.writeVInt(isValid ? 1 : 0, buffer); - BytesUtil.writeByteArray(TupleFilterSerializer.serialize(columnContainerFilter, cs), buffer); - - byte[] result = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); - return result; } @Override - void deserialize(byte[] bytes, IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.wrap(bytes); + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { this.name = BytesUtil.readUTFString(buffer); this.initMethod(); this.colPosition = BytesUtil.readVInt(buffer); this.isValid = BytesUtil.readVInt(buffer) == 1; - - byte[] columnFilter = BytesUtil.readByteArray(buffer); - if (columnFilter != null) { - this.columnContainerFilter = TupleFilterSerializer.deserialize(columnFilter, cs); - } } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java new file mode 100644 index 0000000..d3d5076 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +public interface ITupleFilterTransformer { + TupleFilter transform(TupleFilter tupleFilter); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java deleted file mode 100644 index aed284c..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.metadata.filter; - -/** - * Created by dongli on 1/7/16. - */ -public interface ITupleFilterTranslator { - TupleFilter translate(TupleFilter tupleFilter); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java index 0929cf1..61657fb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.filter; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -46,7 +47,6 @@ public class LogicalTupleFilter extends TupleFilter { return cloneTuple; } - // private TupleFilter reverseNestedNots(TupleFilter filter, int depth) { // if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT)) { // assert (filter.children.size() == 1); @@ -60,7 +60,6 @@ public class LogicalTupleFilter extends TupleFilter { // } // } - @Override public TupleFilter reverse() { switch (operator) { @@ -151,12 +150,13 @@ public class LogicalTupleFilter extends TupleFilter { } @Override - public byte[] serialize(IFilterCodeSystem cs) { - return new byte[0]; + public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) { + //do nothing } @Override - public void deserialize(byte[] bytes, IFilterCodeSystem cs) { + public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) { + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 944ddd0..1e23499 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.filter; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -204,9 +205,9 @@ public abstract class TupleFilter { public abstract Collection getValues(); - abstract byte[] serialize(IFilterCodeSystem cs); + abstract void serialize(IFilterCodeSystem cs,ByteBuffer buffer); - abstract void deserialize(byte[] bytes, IFilterCodeSystem cs); + abstract void deserialize(IFilterCodeSystem cs,ByteBuffer buffer); public static boolean isEvaluableRecursively(TupleFilter filter) { if (filter == null) http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java index a394a51..39ccb15 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -18,12 +18,15 @@ package org.apache.kylin.metadata.filter; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Stack; import org.apache.kylin.common.util.BytesUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf @@ -33,6 +36,8 @@ import org.apache.kylin.common.util.BytesUtil; */ public class TupleFilterSerializer { + private static final Logger logger = LoggerFactory.getLogger(TupleFilterSerializer.class); + public interface Decorator { TupleFilter onSerialize(TupleFilter filter); } @@ -51,8 +56,18 @@ public class TupleFilterSerializer { } public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); - internalSerialize(rootFilter, decorator, buffer, cs); + ByteBuffer buffer; + int bufferSize = BUFFER_SIZE; + while (true) { + try { + buffer = ByteBuffer.allocate(bufferSize); + internalSerialize(rootFilter, decorator, buffer, cs); + break; + } catch (BufferOverflowException e) { + logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize); + bufferSize *= 4; + } + } byte[] result = new byte[buffer.position()]; System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); return result; @@ -86,10 +101,9 @@ public class TupleFilterSerializer { if (flag < 0) { BytesUtil.writeVInt(-1, buffer); } else { - byte[] bytes = filter.serialize(cs); int opVal = filter.getOperator().getValue(); BytesUtil.writeVInt(opVal, buffer); - BytesUtil.writeByteArray(bytes, buffer); + filter.serialize(cs, buffer); BytesUtil.writeVInt(flag, buffer); } } @@ -107,8 +121,7 @@ public class TupleFilterSerializer { // deserialize filter TupleFilter filter = createTupleFilter(opVal); - byte[] filterBytes = BytesUtil.readByteArray(buffer); - filter.deserialize(filterBytes, cs); + filter.deserialize(cs, buffer); if (rootFilter == null) { // push root to stack http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java index 01d3041..5208ba7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -8,11 +8,11 @@ import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.DictCodeSystem; import org.apache.kylin.dict.IDictionaryAware; -import org.apache.kylin.dict.TupleFilterFunctionTranslator; +import org.apache.kylin.dict.TupleFilterFunctionTransformer; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilterSerializer; import org.apache.kylin.metadata.model.TblColRef; @@ -131,8 +131,8 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { if (filter == null) return null; - ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(columnIO.getIDictionaryAware()); - filter = translator.translate(filter); + ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(columnIO.getIDictionaryAware()); + filter = translator.transform(filter); // un-evaluatable filter is replaced with TRUE if (!filter.isEvaluable()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index e96c602..3f00566 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -23,7 +23,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.dict.TupleFilterFunctionTranslator; +import org.apache.kylin.dict.TupleFilterFunctionTransformer; import org.apache.kylin.gridtable.EmptyGTScanner; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; @@ -33,7 +33,7 @@ import org.apache.kylin.gridtable.GTScanRequest; import org.apache.kylin.gridtable.GTUtil; import org.apache.kylin.gridtable.IGTScanner; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -63,9 +63,9 @@ public class CubeSegmentScanner implements IGTScanner { CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping(); - // translate FunctionTupleFilter to IN clause - ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(this.cubeSeg); - filter = translator.translate(filter); + // transform FunctionTupleFilter to equivalent IN clause + ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(this.cubeSeg); + filter = translator.transform(filter); //replace the constant values in filter to dictionary codes TupleFilter gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, mapping.getCuboidDimensionsInGTOrder(), groups); http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java index 92e9699..e7ed1a8 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java @@ -18,8 +18,12 @@ package org.apache.kylin.storage.hbase.common.coprocessor; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collection; +import java.util.Date; import java.util.List; import java.util.Random; @@ -39,6 +43,8 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.Tuple; import org.apache.kylin.metadata.tuple.TupleInfo; +import com.google.common.collect.Lists; + /** * @author xjiang * @@ -64,7 +70,7 @@ public class FilterBaseTest { return groups; } - protected CompareTupleFilter buildCompareFilter(List groups, int index) { + protected CompareTupleFilter buildEQCompareFilter(List groups, int index) { TblColRef column = groups.get(index); CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ); ColumnTupleFilter columnFilter = new ColumnTupleFilter(column); @@ -79,9 +85,31 @@ public class FilterBaseTest { return compareFilter; } + protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException { + CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN); + ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn); + compareFilter.addChild(columnFilter); + + List inValues = Lists.newArrayList(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + Date startDate = simpleDateFormat.parse("1970-01-01"); + Date endDate = simpleDateFormat.parse("2100-01-01"); + Calendar start = Calendar.getInstance(); + start.setTime(startDate); + Calendar end = Calendar.getInstance(); + end.setTime(endDate); + for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) { + inValues.add(simpleDateFormat.format(date)); + } + + ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues); + compareFilter.addChild(constantFilter); + return compareFilter; + } + protected TupleFilter buildAndFilter(List columns) { - CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0); - CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1); + CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0); + CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1); LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND); andFilter.addChild(compareFilter1); andFilter.addChild(compareFilter2); @@ -89,8 +117,8 @@ public class FilterBaseTest { } protected TupleFilter buildOrFilter(List columns) { - CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0); - CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1); + CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0); + CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1); LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR); logicFilter.addChild(compareFilter1); logicFilter.addChild(compareFilter2); @@ -105,12 +133,12 @@ public class FilterBaseTest { TupleFilter then0 = new ConstantTupleFilter("0"); caseFilter.addChild(then0); - TupleFilter when1 = buildCompareFilter(groups, 0); + TupleFilter when1 = buildEQCompareFilter(groups, 0); caseFilter.addChild(when1); TupleFilter then1 = new ConstantTupleFilter("1"); caseFilter.addChild(then1); - TupleFilter when2 = buildCompareFilter(groups, 1); + TupleFilter when2 = buildEQCompareFilter(groups, 1); caseFilter.addChild(when2); TupleFilter then2 = new ConstantTupleFilter("2"); caseFilter.addChild(then2); @@ -153,9 +181,9 @@ public class FilterBaseTest { } String str1 = f1.toString(); - System.out.println("f1=" + str1); + //System.out.println("f1=" + str1); String str2 = f2.toString(); - System.out.println("f2=" + str2); + //System.out.println("f2=" + str2); if (!str1.equals(str2)) { throw new IllegalStateException("f1=" + str1 + ", f2=" + str2); } http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java index bde8dd2..aac09b7 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java @@ -38,7 +38,7 @@ public class FilterEvaluateTest extends FilterBaseTest { @Test public void testEvaluate00() { List groups = buildGroups(); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -54,7 +54,7 @@ public class FilterEvaluateTest extends FilterBaseTest { @Test public void testEvaluate01() { List groups = buildGroups(); - TupleFilter filter = buildCompareFilter(groups, 1); + TupleFilter filter = buildEQCompareFilter(groups, 1); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java index 8bf8ecb..0a21598 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java @@ -18,6 +18,7 @@ package org.apache.kylin.storage.hbase.common.coprocessor; +import java.text.ParseException; import java.util.ArrayList; import java.util.List; @@ -40,7 +41,7 @@ public class FilterSerializeTest extends FilterBaseTest { @Test public void testSerialize01() { List groups = buildGroups(); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -51,7 +52,7 @@ public class FilterSerializeTest extends FilterBaseTest { @Test public void testSerialize02() { List groups = buildGroups(); - TupleFilter filter = buildCompareFilter(groups, 1); + TupleFilter filter = buildEQCompareFilter(groups, 1); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -88,7 +89,7 @@ public class FilterSerializeTest extends FilterBaseTest { TblColRef colRef = new TblColRef(column); List groups = new ArrayList(); groups.add(colRef); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -103,7 +104,7 @@ public class FilterSerializeTest extends FilterBaseTest { TblColRef colRef = new TblColRef(column); List groups = new ArrayList(); groups.add(colRef); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -122,7 +123,7 @@ public class FilterSerializeTest extends FilterBaseTest { TblColRef colRef = new TblColRef(column); List groups = new ArrayList(); groups.add(colRef); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -140,7 +141,7 @@ public class FilterSerializeTest extends FilterBaseTest { TblColRef colRef = new TblColRef(column); List groups = new ArrayList(); groups.add(colRef); - TupleFilter filter = buildCompareFilter(groups, 0); + TupleFilter filter = buildEQCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter, CS); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); @@ -203,6 +204,19 @@ public class FilterSerializeTest extends FilterBaseTest { } @Test + public void testSerialize14() throws ParseException { + List groups = buildGroups(); + TupleFilter filter = buildINCompareFilter(groups.get(0)); + + long start = System.currentTimeMillis(); + byte[] bytes = TupleFilterSerializer.serialize(filter, CS); + System.out.println("Size of serialized filter " + bytes.length + ", serialize time: " + (System.currentTimeMillis() - start)); + TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS); + + compareFilter(filter, newFilter); + } + + @Test public void testDynamic() { final CompareTupleFilter compareDynamicFilter = buildCompareDynamicFilter(buildGroups());