incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/9] git commit: Add super query parser with tests.
Date Tue, 19 Feb 2013 01:59:11 GMT
Add super query parser with tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d73a6ee9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d73a6ee9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d73a6ee9

Branch: refs/heads/0.2-dev
Commit: d73a6ee9aabe0d3a7099f8dd493425ba1d921b11
Parents: b75a202
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Feb 17 15:26:48 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Feb 17 15:26:48 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/server/LayoutStrategy.java     |   21 +++
 .../org/apache/blur/server/QueryConverterImpl.java |   11 +-
 .../java/org/apache/blur/server/TableContext.java  |   24 +++
 .../blur/lucene/search/AbstractWrapperQuery.java   |   38 ++++-
 .../org/apache/blur/lucene/search/SuperParser.java |   79 +++++++++
 .../org/apache/blur/lucene/search/SuperQuery.java  |   32 ++++-
 .../apache/blur/lucene/search/SuperParserTest.java |  131 +++++++++++++++
 7 files changed, 324 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java b/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java
new file mode 100644
index 0000000..9cabe85
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java
@@ -0,0 +1,21 @@
+package org.apache.blur.server;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public class LayoutStrategy {
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java b/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java
index 8b363f9..f9c8aff 100644
--- a/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java
+++ b/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java
@@ -20,8 +20,10 @@ import java.io.IOException;
 
 import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.lucene.LuceneVersionConstant;
+import org.apache.blur.lucene.search.ScoreType;
+import org.apache.blur.lucene.search.SuperParser;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.Query;
 
 public class QueryConverterImpl extends Configured implements QueryConverter {
@@ -29,10 +31,15 @@ public class QueryConverterImpl extends Configured implements QueryConverter
{
   @Override
   public Query convert(org.apache.blur.thrift.generated.Query query) throws IOException {
     TableContext tableContext = getTableContext();
+
     switch (query.getType()) {
     case STRING:
+      ScoreType defaultScoreType = tableContext.getDefaultScoreType();
+      Term defaultPrimeDocTerm = tableContext.getDefaultPrimeDocTerm();
       BlurAnalyzer analyzer = tableContext.getAnalyzer();
-      QueryParser queryParser = new QueryParser(LuceneVersionConstant.LUCENE_VERSION, tableContext.getDefaultFieldName(),
analyzer);
+      String defaultFieldName = tableContext.getDefaultFieldName();
+      
+      SuperParser queryParser = new SuperParser(LuceneVersionConstant.LUCENE_VERSION, defaultFieldName,
analyzer, defaultScoreType, defaultPrimeDocTerm);
       try {
         return queryParser.parse(query.getQueryString());
       } catch (ParseException e) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 894f1c0..10285ef 100644
--- a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -28,12 +28,14 @@ import java.util.Map.Entry;
 import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.lucene.search.ScoreType;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.Similarity;
 
@@ -57,6 +59,10 @@ public class TableContext {
   private long timeBetweenRefreshs;
   private TypeManager typeChecker;
 
+  private ScoreType defaultScoreType;
+
+  private Term defaultPrimeDocTerm;
+
   public TypeManager getTypeManager() {
     return typeChecker;
   }
@@ -86,6 +92,8 @@ public class TableContext {
     tableContext.timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS,
60000);
     tableContext.timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS,
5000);
     tableContext.typeChecker = typeChecker;
+    tableContext.defaultPrimeDocTerm = new Term("_primedoc_");
+    tableContext.defaultScoreType = ScoreType.SUPER;
 
     Class<?> c1 = configuration.getClass(BLUR_LUCENE_INDEX_DELETION_POLICY_CLASS, KeepOnlyLastCommitDeletionPolicy.class);
     tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1,
configuration), tableContext);
@@ -198,4 +206,20 @@ public class TableContext {
   public void setDefaultFieldName(String defaultFieldName) {
     this.defaultFieldName = defaultFieldName;
   }
+
+  public Term getDefaultPrimeDocTerm() {
+    return defaultPrimeDocTerm;
+  }
+
+  public void setDefaultScoreType(ScoreType defaultScoreType) {
+    this.defaultScoreType = defaultScoreType;
+  }
+
+  public void setDefaultPrimeDocTerm(Term defaultPrimeDocTerm) {
+    this.defaultPrimeDocTerm = defaultPrimeDocTerm;
+  }
+
+  public ScoreType getDefaultScoreType() {
+    return defaultScoreType;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
b/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
index 9f31f3d..8f2b2cc 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
@@ -51,10 +51,6 @@ public abstract class AbstractWrapperQuery extends Query {
 
   public abstract Weight createWeight(IndexSearcher searcher) throws IOException;
 
-  public boolean equals(Object obj) {
-    return _query.equals(obj);
-  }
-
   public void extractTerms(Set<Term> terms) {
     _query.extractTerms(terms);
   }
@@ -67,16 +63,42 @@ public abstract class AbstractWrapperQuery extends Query {
     return searcher.getSimilarity();
   }
 
-  public int hashCode() {
-    return _query.hashCode();
-  }
-
   public abstract Query rewrite(IndexReader reader) throws IOException;
 
   public void setBoost(float b) {
     _query.setBoost(b);
   }
 
+  
+  
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = super.hashCode();
+    result = prime * result + ((_query == null) ? 0 : _query.hashCode());
+    result = prime * result + (_rewritten ? 1231 : 1237);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (!super.equals(obj))
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    AbstractWrapperQuery other = (AbstractWrapperQuery) obj;
+    if (_query == null) {
+      if (other._query != null)
+        return false;
+    } else if (!_query.equals(other._query))
+      return false;
+    if (_rewritten != other._rewritten)
+      return false;
+    return true;
+  }
+
   public abstract String toString();
 
   public abstract String toString(String field);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
new file mode 100644
index 0000000..80a53fa
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
@@ -0,0 +1,79 @@
+package org.apache.blur.lucene.search;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Version;
+
+public class SuperParser extends QueryParser {
+
+  private static final String MUST_NOT_STRING = "-";
+  private static final String MUST_STRING = "+";
+  private static final Pattern PATTERN = Pattern.compile("([-+]{0,1})\\s*?super\\s*?\\:\\s*?\\<(.*?)\\>");
+  private static final Pattern CHECK = Pattern.compile("super\\s*?\\:\\s*?\\<");
+
+  private final Analyzer a;
+  private final String f;
+  private final Version matchVersion;
+  private final Term defaultPrimeDocTerm;
+  private final ScoreType defaultScoreType;
+
+  public SuperParser(Version matchVersion, String f, Analyzer a, ScoreType defaultScoreType,
Term defaultPrimeDocTerm) {
+    super(matchVersion, f, a);
+    this.matchVersion = matchVersion;
+    this.f = f;
+    this.a = a;
+    this.defaultPrimeDocTerm = defaultPrimeDocTerm;
+    this.defaultScoreType = defaultScoreType;
+  }
+
+  @Override
+  public Query parse(String query) throws ParseException {
+    Matcher matcher = PATTERN.matcher(query);
+    BooleanQuery booleanQuery = null;
+    while (matcher.find()) {
+      int count = matcher.groupCount();
+      for (int i = 0; i < count; i++) {
+        String occurString = matcher.group(i + 1);
+        i++;
+        String superQueryStr = matcher.group(i + 1);
+        Matcher matcherCheck = CHECK.matcher(superQueryStr);
+        if (matcherCheck.find()) {
+          throw new ParseException("Embedded super queries are not allowed [" + query + "].");
+        }
+
+        if (booleanQuery == null) {
+          booleanQuery = new BooleanQuery();
+        }
+
+        Occur occur = getOccur(occurString);
+        QueryParser parser = new QueryParser(matchVersion, f, a);
+
+        Query superQuery = parser.parse(superQueryStr);
+        booleanQuery.add(new SuperQuery(superQuery, defaultScoreType, defaultPrimeDocTerm),
occur);
+      }
+    }
+    if (booleanQuery == null) {
+      return super.parse(query);
+    }
+    return booleanQuery;
+  }
+
+  private Occur getOccur(String occurString) {
+    if (occurString.equals(MUST_STRING)) {
+      return Occur.MUST;
+    }
+    if (occurString.equals(MUST_NOT_STRING)) {
+      return Occur.MUST_NOT;
+    }
+    return Occur.SHOULD;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
index 46f62b1..cc0d484 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
@@ -74,11 +74,39 @@ public class SuperQuery extends AbstractWrapperQuery {
   }
 
   public String toString() {
-    return "super:{" + _query.toString() + "}";
+    return "super:<" + _query.toString() + ">";
   }
 
   public String toString(String field) {
-    return "super:{" + _query.toString(field) + "}";
+    return "super:<" + _query.toString(field) + ">";
+  }
+  
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = super.hashCode();
+    result = prime * result + ((primeDocTerm == null) ? 0 : primeDocTerm.hashCode());
+    result = prime * result + ((scoreType == null) ? 0 : scoreType.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (!super.equals(obj))
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    SuperQuery other = (SuperQuery) obj;
+    if (primeDocTerm == null) {
+      if (other.primeDocTerm != null)
+        return false;
+    } else if (!primeDocTerm.equals(other.primeDocTerm))
+      return false;
+    if (scoreType != other.scoreType)
+      return false;
+    return true;
   }
 
   public static class SuperWeight extends Weight {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
new file mode 100644
index 0000000..381ffe1
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
@@ -0,0 +1,131 @@
+package org.apache.blur.lucene.search;
+
+import static org.junit.Assert.*;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SuperParserTest {
+
+  private SuperParser parser;
+
+  @Before
+  public void setup() {
+    parser = new SuperParser(Version.LUCENE_40, "", new WhitespaceAnalyzer(Version.LUCENE_40),
ScoreType.SUPER, new Term("_primedoc_"));
+  }
+
+  @Test
+  public void testParser1() throws ParseException {
+    Query query = parser.parse(" +super:<a:a d:e b:b> ");
+
+    BooleanQuery booleanQuery = new BooleanQuery();
+    booleanQuery.add(new TermQuery(new Term("a", "a")), Occur.SHOULD);
+    booleanQuery.add(new TermQuery(new Term("d", "e")), Occur.SHOULD);
+    booleanQuery.add(new TermQuery(new Term("b", "b")), Occur.SHOULD);
+    SuperQuery superQuery = new SuperQuery(booleanQuery, ScoreType.SUPER, new Term("_primedoc_"));
+
+    BooleanQuery bq = new BooleanQuery();
+    bq.add(superQuery, Occur.MUST);
+    
+    assertEquals(bq, query);
+
+  }
+
+  @Test
+  public void testParser2() throws ParseException {
+    Query query = parser.parse("super:<c:c d:d>");
+
+    BooleanQuery booleanQuery = new BooleanQuery();
+    booleanQuery.add(new TermQuery(new Term("c", "c")), Occur.SHOULD);
+    booleanQuery.add(new TermQuery(new Term("d", "d")), Occur.SHOULD);
+    SuperQuery superQuery = new SuperQuery(booleanQuery, ScoreType.SUPER, new Term("_primedoc_"));
+
+    BooleanQuery bq = new BooleanQuery();
+    bq.add(superQuery, Occur.SHOULD);
+
+    assertEquals(bq, query);
+  }
+
+  @Test
+  public void testParser3() throws ParseException {
+    Query query = parser.parse("a:a d:e b:b");
+
+    BooleanQuery booleanQuery = new BooleanQuery();
+    booleanQuery.add(new TermQuery(new Term("a", "a")), Occur.SHOULD);
+    booleanQuery.add(new TermQuery(new Term("d", "e")), Occur.SHOULD);
+    booleanQuery.add(new TermQuery(new Term("b", "b")), Occur.SHOULD);
+
+    assertEquals(booleanQuery, query);
+  }
+
+  @Test
+  public void testParser4() throws ParseException {
+    Query query = parser.parse("super:<a:a d:e b:b>  - super:<c:c d:d>");
+
+    BooleanQuery booleanQuery1 = new BooleanQuery();
+    booleanQuery1.add(new TermQuery(new Term("a", "a")), Occur.SHOULD);
+    booleanQuery1.add(new TermQuery(new Term("d", "e")), Occur.SHOULD);
+    booleanQuery1.add(new TermQuery(new Term("b", "b")), Occur.SHOULD);
+
+    BooleanQuery booleanQuery2 = new BooleanQuery();
+    booleanQuery2.add(new TermQuery(new Term("c", "c")), Occur.SHOULD);
+    booleanQuery2.add(new TermQuery(new Term("d", "d")), Occur.SHOULD);
+
+    SuperQuery superQuery1 = new SuperQuery(booleanQuery1, ScoreType.SUPER, new Term("_primedoc_"));
+    SuperQuery superQuery2 = new SuperQuery(booleanQuery2, ScoreType.SUPER, new Term("_primedoc_"));
+
+    BooleanQuery booleanQuery = new BooleanQuery();
+    booleanQuery.add(superQuery1, Occur.SHOULD);
+    booleanQuery.add(superQuery2, Occur.MUST_NOT);
+
+    assertEquals(booleanQuery, query);
+  }
+
+  @Test
+  public void testParser5() throws ParseException {
+
+    Query query = parser.parse("super:<a:a d:{e TO f} b:b test:hello\\<> - super:<c:c
d:d>");
+
+    BooleanQuery booleanQuery1 = new BooleanQuery();
+    booleanQuery1.add(new TermQuery(new Term("a", "a")), Occur.SHOULD);
+    booleanQuery1.add(new TermRangeQuery("d", new BytesRef("e"), new BytesRef("f"), false,
false), Occur.SHOULD);
+    booleanQuery1.add(new TermQuery(new Term("b", "b")), Occur.SHOULD);
+    // std analyzer took the "<" out
+    booleanQuery1.add(new TermQuery(new Term("test", "hello<")), Occur.SHOULD);
+
+    BooleanQuery booleanQuery2 = new BooleanQuery();
+    booleanQuery2.add(new TermQuery(new Term("c", "c")), Occur.SHOULD);
+    booleanQuery2.add(new TermQuery(new Term("d", "d")), Occur.SHOULD);
+
+    SuperQuery superQuery1 = new SuperQuery(booleanQuery1, ScoreType.SUPER, new Term("_primedoc_"));
+    SuperQuery superQuery2 = new SuperQuery(booleanQuery2, ScoreType.SUPER, new Term("_primedoc_"));
+
+    BooleanQuery booleanQuery = new BooleanQuery();
+    booleanQuery.add(superQuery1, Occur.SHOULD);
+    booleanQuery.add(superQuery2, Occur.MUST_NOT);
+
+    assertEquals(booleanQuery, query);
+  }
+
+  @Test
+  public void testParser6() throws ParseException {
+    SuperParser parser = new SuperParser(Version.LUCENE_40, "", new StandardAnalyzer(Version.LUCENE_40),
ScoreType.SUPER, new Term("_primedoc_"));
+    try {
+      parser.parse("super : <a:a d:{e TO d} b:b super:<test:hello\\<>> super:<c:c
d:d>");
+      fail();
+    } catch (ParseException e) {
+      // should throw an error
+    }
+  }
+}


Mime
View raw message