carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [HOTFIX] Fix lucene match limit code
Date Sun, 06 May 2018 11:57:55 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 9db662a2d -> 5cad92f4f


[HOTFIX] Fix lucene match limit code

Problem
Currently, Lucene match limit is set as Static in MatchExpression it cannot work in concurrent
scenarios.
Solution:
Change to object variable and get the match max limit from expression.

This closes #2250


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5cad92f4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5cad92f4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5cad92f4

Branch: refs/heads/master
Commit: 5cad92f4f23f47f6c0b660b69f2c6edad8597974
Parents: 9db662a
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Sun Apr 29 20:49:18 2018 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Sun May 6 19:57:42 2018 +0800

----------------------------------------------------------------------
 .../core/scan/expression/MatchExpression.java      | 15 ++++++++++-----
 .../datamap/lucene/LuceneFineGrainDataMap.java     | 17 ++++++++++++++---
 .../apache/spark/sql/optimizer/CarbonFilters.scala |  6 +++---
 3 files changed, 27 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
index db806a7..52104be 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
@@ -29,15 +29,20 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf;
 @InterfaceAudience.Internal
 public class MatchExpression extends Expression {
   private String queryString;
-  public static String maxDoc;
+  private int maxDoc;
 
-  public MatchExpression(String queryString, String maxDoc) {
+  public MatchExpression(String queryString) {
     this.queryString = queryString;
-    setMaxDoc(maxDoc);
+    this.maxDoc = -1;
   }
 
-  private void setMaxDoc(String maxDoc) {
-    MatchExpression.maxDoc = maxDoc;
+  public MatchExpression(String queryString, int maxDoc) {
+    this.queryString = queryString;
+    this.maxDoc = maxDoc;
+  }
+
+  public int getMaxDoc() {
+    return maxDoc;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
index 86fba32..8e457b7 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
@@ -153,8 +153,19 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap {
    * Return Maximum records
    * @return
    */
-  private int getMaxDoc() {
-    return Integer.parseInt(MatchExpression.maxDoc);
+  private int getMaxDoc(Expression expression) {
+    if (expression.getFilterExpressionType() == ExpressionType.TEXT_MATCH) {
+      int maxDoc = ((MatchExpression) expression).getMaxDoc();
+      if (maxDoc < 0) {
+        maxDoc = Integer.MAX_VALUE;
+      }
+      return maxDoc;
+    }
+
+    for (Expression child : expression.getChildren()) {
+      return getMaxDoc(child);
+    }
+    return Integer.MAX_VALUE;
   }
 
   /**
@@ -172,7 +183,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap {
     String strQuery = getQueryString(filterExp.getFilterExpression());
     int maxDocs;
     try {
-      maxDocs = getMaxDoc();
+      maxDocs = getMaxDoc(filterExp.getFilterExpression());
     } catch (NumberFormatException e) {
       maxDocs = Integer.MAX_VALUE;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 1e46998..07a444f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.optimizer
 import java.util
 
 import scala.collection.JavaConverters._
+import scala.util.Try
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions._
@@ -30,7 +31,6 @@ import org.apache.spark.sql.CarbonEndsWith
 import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.hive.CarbonSessionCatalog
-import org.apache.spark.sql.sources.Filter
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datamap.Segment
@@ -141,9 +141,9 @@ object CarbonFilters {
         case FalseExpr() =>
           Some(new FalseExpression(null))
         case TextMatch(queryString) =>
-          Some(new MatchExpression(queryString, null))
+          Some(new MatchExpression(queryString))
         case TextMatchLimit(queryString, maxDoc) =>
-          Some(new MatchExpression(queryString, maxDoc))
+          Some(new MatchExpression(queryString, Try(maxDoc.toInt).getOrElse(Integer.MAX_VALUE)))
         case _ => None
       }
     }


Mime
View raw message