kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [29/50] [abbrv] kylin git commit: minor, add LIKE FilterOperatorEnum in case it will be pushed down(it won't for now)
Date Sat, 03 Sep 2016 06:37:36 GMT
minor, add LIKE FilterOperatorEnum in case it will be pushed down(it won't for now)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e0b8d80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e0b8d80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e0b8d80

Branch: refs/heads/1.5.x-HBase1.x
Commit: 9e0b8d8033dbe4e86d299adbd102cf52792cd5fe
Parents: c5fdacd
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Wed Aug 31 11:47:37 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Wed Aug 31 11:47:42 2016 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/gridtable/GTUtil.java | 53 +++++++++++
 .../filter/BuiltInFunctionTupleFilter.java      | 22 ++++-
 .../filter/EvaluatableLikeFunction.java         | 96 ++++++++++++++++++++
 .../EvaluatableLikeFunctionTransformer.java     | 76 ++++++++++++++++
 .../kylin/metadata/filter/TupleFilter.java      |  2 +-
 .../metadata/filter/TupleFilterSerializer.java  |  3 +
 .../org/apache/kylin/query/KylinTestBase.java   |  8 +-
 7 files changed, 254 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 4b50176..ce0f016 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.gridtable;
 
+import java.lang.reflect.InvocationTargetException;
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.List;
@@ -25,6 +26,7 @@ import java.util.Set;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -104,6 +106,13 @@ public class GTUtil {
                 if (encodeConstants && filter instanceof CompareTupleFilter) {
                     return encodeConstants((CompareTupleFilter) filter);
                 }
+                if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter)
{
+                    if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
+                        return encodeConstants((BuiltInFunctionTupleFilter) filter);
+                    } else {
+                        throw new IllegalStateException("Nested BuiltInFunctionTupleFilter
is not supported to be pushed down");
+                    }
+                }
 
                 return filter;
             }
@@ -122,9 +131,13 @@ public class GTUtil {
                     return oldCompareFilter;
                 }
 
+                //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach
here caz it will be transformed by BuiltInFunctionTransformer
                 CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator());
                 newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
 
+                //for CompareTupleFilter containing dynamicVariables, the below codes will
actually replace dynamicVariables
+                //with normal ConstantTupleFilter
+
                 Object firstValue = constValues.iterator().next();
                 int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex()
: colMapping.indexOf(externalCol);
 
@@ -199,6 +212,46 @@ public class GTUtil {
                 return result;
             }
 
+            @SuppressWarnings({ "rawtypes", "unchecked" })
+            private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
+                // extract ColumnFilter & ConstantFilter
+                TblColRef externalCol = funcFilter.getColumn();
+
+                if (externalCol == null) {
+                    return funcFilter;
+                }
+
+                Collection constValues = funcFilter.getConstantTupleFilter().getValues();
+                if (constValues == null || constValues.isEmpty()) {
+                    return funcFilter;
+                }
+
+                BuiltInFunctionTupleFilter newFuncFilter;
+                try {
+                    newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
+                } catch (InstantiationException | IllegalAccessException | NoSuchMethodException
| InvocationTargetException e) {
+                    throw new RuntimeException(e);
+                }
+                newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
+
+                int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex()
: colMapping.indexOf(externalCol);
+
+                ByteArray code;
+
+                // translate constant into code
+                Set newValues = Sets.newHashSet();
+                for (Object value : constValues) {
+                    code = translate(col, value, 0);
+                    if (code == null) {
+                        throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
+                    }
+                    newValues.add(code);
+                }
+                newFuncFilter.addChild(new ConstantTupleFilter(newValues));
+
+                return newFuncFilter;
+            }
+
             transient ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
 
             private ByteArray translate(int col, Object value, int roundingFlag) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 5a10371..40afb18 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.metadata.filter.function.BuiltInMethod;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -40,13 +41,18 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
     protected String name;
     // FIXME Only supports single parameter functions currently
     protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case)
or FunctionTupleFilter(complex case like substr(lower()))
+    protected ConstantTupleFilter constantTupleFilter;
     protected int colPosition;
     protected Method method;
     protected List<Serializable> methodParams;
     protected boolean isValidFunc = false;
 
     public BuiltInFunctionTupleFilter(String name) {
-        super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
+        this(name, null);
+    }
+
+    public BuiltInFunctionTupleFilter(String name, FilterOperatorEnum filterOperatorEnum)
{
+        super(Lists.<TupleFilter> newArrayList(), filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION
: filterOperatorEnum);
         this.methodParams = Lists.newArrayList();
 
         if (name != null) {
@@ -59,6 +65,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
         return name;
     }
 
+    public ConstantTupleFilter getConstantTupleFilter() {
+        return constantTupleFilter;
+    }
+
     public TblColRef getColumn() {
         if (columnContainerFilter == null)
             return null;
@@ -71,6 +81,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
         throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
     }
 
+    public boolean hasNested() {
+        return (columnContainerFilter != null && columnContainerFilter instanceof
BuiltInFunctionTupleFilter);
+    }
+
     public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException
{
         if (columnContainerFilter instanceof ColumnTupleFilter)
             methodParams.set(colPosition, (Serializable) input);
@@ -90,6 +104,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
             colPosition = methodParams.size();
             methodParams.add(null);
         } else if (child instanceof ConstantTupleFilter) {
+            this.constantTupleFilter = (ConstantTupleFilter) child;
             Serializable constVal = (Serializable) child.getValues().iterator().next();
             try {
                 Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
@@ -122,12 +137,13 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter
{
 
     @Override
     public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
-        throw new UnsupportedOperationException("Function filter cannot serialized");
+        BytesUtil.writeUTFString(name, buffer);
     }
 
     @Override
     public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
-        throw new UnsupportedOperationException("Function filter cannot serialized");
+        this.name = BytesUtil.readUTFString(buffer);
+        this.initMethod();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
new file mode 100644
index 0000000..59961ac
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.StringSerializer;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
+public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
+
+    public EvaluatableLikeFunction(String name) {
+        super(name, FilterOperatorEnum.LIKE);
+    }
+
+    @Override
+    public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
+
+        // extract tuple value
+        Object tupleValue = null;
+        for (TupleFilter filter : this.children) {
+            if (!isConstant(filter)) {
+                filter.evaluate(tuple, cs);
+                tupleValue = filter.getValues().iterator().next();
+            }
+        }
+
+        // consider null case
+        if (cs.isNull(tupleValue)) {
+            return false;
+        }
+
+        ByteArray valueByteArray = (ByteArray) tupleValue;
+        StringSerializer serializer = new StringSerializer(DataType.getType("string"));
+        String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(),
valueByteArray.length()));
+        try {
+            return (Boolean) invokeFunction(value);
+        } catch (InvocationTargetException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+        if (!isValid()) {
+            throw new IllegalStateException("must be valid");
+        }
+        if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1)
== null) {
+            throw new IllegalArgumentException("bad methodParams: " + methodParams);
+        }
+        BytesUtil.writeUTFString(name, buffer);
+    }
+
+    @Override
+    public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+        this.name = BytesUtil.readUTFString(buffer);
+        this.initMethod();
+    }
+
+    @Override
+    public boolean isEvaluable() {
+        return true;
+    }
+
+    private boolean isConstant(TupleFilter filter) {
+        return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+    }
+
+    public String getLikePattern() {
+        ByteArray byteArray = (ByteArray) methodParams.get(1);
+        StringSerializer s = new StringSerializer(DataType.getType("string"));
+        String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(),
byteArray.length()));
+        return pattern;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
new file mode 100644
index 0000000..7295533
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 
+ */
+public class EvaluatableLikeFunctionTransformer {
+    public static final Logger logger = LoggerFactory.getLogger(EvaluatableLikeFunctionTransformer.class);
+
+    public static TupleFilter transform(TupleFilter tupleFilter) {
+        TupleFilter translated = null;
+        if (tupleFilter instanceof CompareTupleFilter) {
+            CompareTupleFilter compTupleFilter = (CompareTupleFilter) tupleFilter;
+            if (compTupleFilter.getFunction() != null && (compTupleFilter.getFunction()
instanceof BuiltInFunctionTupleFilter)) {
+                throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported
:" + ((BuiltInFunctionTupleFilter) compTupleFilter.getFunction()).getName());
+            }
+        } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
+            BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter)
tupleFilter;
+            if (isLikeFunction(builtInFunctionTupleFilter)) {
+                for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+                    if (!(child instanceof ColumnTupleFilter) && !(child instanceof
ConstantTupleFilter)) {
+                        throw new IllegalArgumentException("Only simple like clause is supported");
+                    }
+                }
+
+                translated = new EvaluatableLikeFunction(builtInFunctionTupleFilter.getName());
+                for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+                    translated.addChild(child);
+                }
+
+            } else {
+                throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported:
" + builtInFunctionTupleFilter.getName());
+            }
+
+        } else if (tupleFilter instanceof LogicalTupleFilter) {
+            @SuppressWarnings("unchecked")
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
tupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter transformed = transform(childIterator.next());
+                if (transformed != null) {
+                    childIterator.set(transformed);
+                } else {
+                    throw new IllegalStateException("Should not be null");
+                }
+            }
+        }
+        return translated == null ? tupleFilter : translated;
+    }
+
+    private static boolean isLikeFunction(BuiltInFunctionTupleFilter builtInFunctionTupleFilter)
{
+        return "like".equalsIgnoreCase(builtInFunctionTupleFilter.getName());
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 3250640..2fb4e1f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -38,7 +38,7 @@ import com.google.common.collect.Maps;
 public abstract class TupleFilter {
 
     public enum FilterOperatorEnum {
-        EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10),
AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35),
MASSIN(36);
+        EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10),
AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35),
MASSIN(36), LIKE(37);
 
         private final int value;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index f6c8bc4..0aca9a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -188,6 +188,9 @@ public class TupleFilterSerializer {
         case FUNCTION:
             filter = new BuiltInFunctionTupleFilter(null);
             break;
+        case LIKE:
+            filter = new EvaluatableLikeFunction(null);
+            break;
         case MASSIN:
             filter = new MassInTupleFilter();
             break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 4e59815..2ad1105 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -63,6 +63,8 @@ import com.google.common.io.Files;
  */
 public class KylinTestBase {
 
+    public static boolean PRINT_RESULT = false;
+
     class ObjectArray {
         Object[] data;
 
@@ -224,7 +226,8 @@ public class KylinTestBase {
         if (needSort) {
             queryTable = new SortedTable(queryTable, columnNames);
         }
-        //printResult(queryTable);
+        if (PRINT_RESULT)
+            printResult(queryTable);
 
         return queryTable;
     }
@@ -280,7 +283,8 @@ public class KylinTestBase {
         if (needSort) {
             queryTable = new SortedTable(queryTable, columnNames);
         }
-        //printResult(queryTable);
+        if (PRINT_RESULT)
+            printResult(queryTable);
         return queryTable;
     }
 


Mime
View raw message