carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [17/20] carbondata git commit: [CARBONDATA-2206] Fixed lucene datamap evaluation issue in executor
Date Sun, 04 Mar 2018 14:57:59 GMT
[CARBONDATA-2206] Fixed lucene datamap evaluation issue in executor

In case of MatchExpression it should return same bitset from RowLevelFilterExecuterImpl

This closes #2010


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8049185c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8049185c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8049185c

Branch: refs/heads/datamap-rebase1
Commit: 8049185cbdb001588be5c5d97f73bac035d16d6c
Parents: c0133aa
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Tue Feb 27 19:14:15 2018 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Sun Mar 4 22:45:17 2018 +0800

----------------------------------------------------------------------
 .../carbondata/core/datamap/DataMapChooser.java |  3 +-
 .../core/scan/expression/MatchExpression.java   | 57 ++++++++++++++++++++
 .../executer/RowLevelFilterExecuterImpl.java    |  4 ++
 .../lucene/LuceneFineGrainDataMapSuite.scala    |  8 ++-
 .../datamap/expression/MatchExpression.java     | 56 -------------------
 .../spark/sql/optimizer/CarbonFilters.scala     |  3 +-
 6 files changed, 67 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index c8c971d..f9214a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -228,7 +228,8 @@ public class DataMapChooser {
 
   private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions,
       Set<ExpressionType> expressionTypes) {
-    if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH)) {
+    if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH) &&
+        expressionTypes.contains(ExpressionType.TEXT_MATCH)) {
       // TODO: fix it with right logic
       return true;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
new file mode 100644
index 0000000..3677b51
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+@InterfaceAudience.Internal
+public class MatchExpression extends Expression {
+  private String queryString;
+
+  public MatchExpression(String queryString) {
+    this.queryString = queryString;
+  }
+
+  @Override
+  public ExpressionResult evaluate(RowIntf value)
+      throws FilterUnsupportedException, FilterIllegalMemberException {
+    return new ExpressionResult(DataTypes.BOOLEAN,true);
+  }
+
+  @Override
+  public ExpressionType getFilterExpressionType() {
+    return ExpressionType.TEXT_MATCH;
+  }
+
+  @Override
+  public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+
+  }
+
+  @Override
+  public String getString() {
+    return queryString;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 540607d..2bd49ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -50,6 +50,7 @@ import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
 import org.apache.carbondata.core.scan.executor.util.QueryUtil;
 import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
 import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.MatchExpression;
 import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
@@ -191,6 +192,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
   @Override
   public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
       boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+    if (exp instanceof MatchExpression) {
+      return rawBlockletColumnChunks.getBitSetGroup();
+    }
     readColumnChunks(rawBlockletColumnChunks);
     // CHECKSTYLE:ON
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 4766281..5e28e8a 100644
--- a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -61,12 +61,10 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
 
     sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
 
-    sql("SELECT * FROM datamap_test ORDER BY id").show
-
     //    sql("select * from normal_test where name='n34000'").show
-    sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')").show
-    sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')").show
-    sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')").show
+    checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')"), sql(s"select
* from datamap_test where name='n10'"))
+//    checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')"), sql(s"SELECT
* FROM datamap_test WHERE name like 'n10%'"))
+    checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')"), sql(s"SELECT
* FROM datamap_test WHERE city='c020'"))
 
     //    checkAnswer(
     //      sql("select * from datamap_test where match('name:n34000')"),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
b/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
deleted file mode 100644
index fceb729..0000000
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.expression;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.ExpressionResult;
-import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-
-@InterfaceAudience.Internal
-public class MatchExpression extends Expression {
-  private String queryString;
-
-  public MatchExpression(String queryString) {
-    this.queryString = queryString;
-  }
-
-  @Override
-  public ExpressionResult evaluate(RowIntf value)
-      throws FilterUnsupportedException, FilterIllegalMemberException {
-    return null;
-  }
-
-  @Override
-  public ExpressionType getFilterExpressionType() {
-    return ExpressionType.TEXT_MATCH;
-  }
-
-  @Override
-  public void findAndSetChild(Expression oldExpr, Expression newExpr) {
-
-  }
-
-  @Override
-  public String getString() {
-    return queryString;
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8049185c/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 0aedd40..2213fdb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -37,7 +37,7 @@ import org.apache.carbondata.core.indexstore.PartitionSpec
 import org.apache.carbondata.core.metadata.SegmentFileStore
 import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression,
Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression,
Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression}
 import org.apache.carbondata.core.scan.expression.conditional._
 import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression,
OrExpression}
 import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
@@ -46,7 +46,6 @@ import org.apache.carbondata.core.util.ThreadLocalSessionInfo
 import org.apache.carbondata.core.util.path.CarbonTablePath
 import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.datamap.TextMatch
-import org.apache.carbondata.datamap.expression.MatchExpression
 import org.apache.carbondata.spark.CarbonAliasDecoderRelation
 import org.apache.carbondata.spark.util.CarbonScalaUtil
 


Mime
View raw message