incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: The first 4 types have been created (Long, Integer, Float, and Double) they have been integrated into the query parser as well. So they should be able to be used during indexing and querying.
Date Thu, 06 Sep 2012 19:02:05 GMT
Updated Branches:
  refs/heads/master 5152c6cb1 -> af8229b4b


The first 4 types have been created (Long, Integer, Float, and Double) they have been integrated
into the query parser as well.  So they should be able to be used during indexing and querying.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/af8229b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/af8229b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/af8229b4

Branch: refs/heads/master
Commit: af8229b4b6589f4fd9d89f9b6229aef77deb3863
Parents: e5fc222
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Sep 6 15:00:19 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Sep 6 15:00:19 2012 -0400

----------------------------------------------------------------------
 .../org/apache/blur/analysis/BlurAnalyzer.java     |  106 ++++++--
 .../org/apache/blur/lucene/search/SuperParser.java |  166 +++++++-----
 .../org/apache/blur/lucene/search/SuperQuery.java  |    4 +
 .../java/org/apache/blur/manager/IndexManager.java |   12 +-
 .../org/apache/blur/manager/QueryParserUtil.java   |    7 +-
 .../apache/blur/lucene/search/SuperParserTest.java |  219 +++++++++++++++
 .../apache/blur/search/RandomSuperQueryTest.java   |    2 +-
 7 files changed, 412 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
index d6d43c6..3129c0e 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
@@ -51,6 +51,8 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Version;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TJSONProtocol;
@@ -58,18 +60,20 @@ import org.apache.thrift.transport.TMemoryBuffer;
 import org.apache.thrift.transport.TMemoryInputTransport;
 
 public class BlurAnalyzer extends Analyzer {
-  
-  private enum TYPES {
-    LONG, DOUBLE, FLOAT, INTEGER
+
+  public enum TYPE {
+    LONG, DOUBLE, FLOAT, INTEGER, DEFAULT
   }
-  
+
   @SuppressWarnings("serial")
-  private static Set<String> typeNameCache = new HashSet<String>() {{
-    TYPES[] values = TYPES.values();
-    for (TYPES t : values) {
-      add(t.name());
+  private static Set<String> typeNameCache = new HashSet<String>() {
+    {
+      TYPE[] values = TYPE.values();
+      for (TYPE t : values) {
+        add(t.name());
+      }
     }
-  }};
+  };
 
   private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
   public static final BlurAnalyzer BLANK_ANALYZER = new BlurAnalyzer(new KeywordAnalyzer());
@@ -82,7 +86,8 @@ public class BlurAnalyzer extends Analyzer {
   private AnalyzerDefinition _analyzerDefinition;
   private Analyzer _fullTextAnalyzer = new StandardAnalyzer(LUCENE_VERSION);
   private Analyzer _defaultAnalyzer;
-  private HashMap<String, Analyzer> _analyzers = new HashMap<String, Analyzer>();
+  private Map<String, Analyzer> _analyzers = new HashMap<String, Analyzer>();
+  private Map<String, TYPE> _typeLookup = new HashMap<String, BlurAnalyzer.TYPE>();
 
   public Set<String> getSubIndexNames(String indexName) {
     return _subIndexNameLookups.get(indexName);
@@ -98,20 +103,24 @@ public class BlurAnalyzer extends Analyzer {
     ColumnDefinition defaultDefinition = analyzerDefinition.getDefaultDefinition();
     String fullTextAnalyzerClassName = analyzerDefinition.fullTextAnalyzerClassName;
     if (fullTextAnalyzerClassName != null) {
-      _fullTextAnalyzer = getAnalyzerByClassName(fullTextAnalyzerClassName, aliases);
+      _fullTextAnalyzer = getAnalyzerByClassName(fullTextAnalyzerClassName, aliases, null,
null);
     }
     if (defaultDefinition == null) {
       defaultDefinition = new ColumnDefinition(STANDARD, true, null);
       analyzerDefinition.setDefaultDefinition(defaultDefinition);
     }
-    _defaultAnalyzer = getAnalyzerByClassName(defaultDefinition.getAnalyzerClassName(), aliases);
+    _defaultAnalyzer = getAnalyzerByClassName(defaultDefinition.getAnalyzerClassName(), aliases,
null, null);
     KeywordAnalyzer keywordAnalyzer = new KeywordAnalyzer();
     _analyzers = new HashMap<String, Analyzer>();
     _analyzers.put(ROW_ID, keywordAnalyzer);
     _analyzers.put(RECORD_ID, keywordAnalyzer);
     _analyzers.put(PRIME_DOC, keywordAnalyzer);
     _analyzers.put(SUPER, _fullTextAnalyzer);
-    load(_analyzers, _analyzerDefinition.columnFamilyDefinitions, _fullTextFields, _subIndexNameLookups,
_storeMap, _fullTextColumnFamilies);
+    load(_analyzers, _analyzerDefinition.columnFamilyDefinitions, _fullTextFields, _subIndexNameLookups,
_storeMap, _fullTextColumnFamilies, _typeLookup);
+  }
+
+  public BlurAnalyzer() {
+    this(new StandardAnalyzer(LUCENE_VERSION));
   }
 
   private Analyzer getAnalyzer(String name) {
@@ -119,6 +128,46 @@ public class BlurAnalyzer extends Analyzer {
     return analyzer;
   }
 
+  public TYPE getTypeLookup(String field) {
+    TYPE type = _typeLookup.get(field);
+    if (type == null) {
+      return TYPE.DEFAULT;
+    }
+    return type;
+  }
+
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean inclusive)
{
+    Analyzer analyzer = getAnalyzer(field);
+    if (analyzer instanceof LongAnalyzer) {
+      LongAnalyzer a = (LongAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      int radix = a.getRadix();
+      long min = Long.parseLong(part1, radix);
+      long max = Long.parseLong(part2, radix);
+      return NumericRangeQuery.newLongRange(field, precisionStep, min, max, inclusive, inclusive);
+    } else if (analyzer instanceof DoubleAnalyzer) {
+      DoubleAnalyzer a = (DoubleAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      double min = Double.parseDouble(part1);
+      double max = Double.parseDouble(part2);
+      return NumericRangeQuery.newDoubleRange(field, precisionStep, min, max, inclusive,
inclusive);
+    } else if (analyzer instanceof FloatAnalyzer) {
+      FloatAnalyzer a = (FloatAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      float min = Float.parseFloat(part1);
+      float max = Float.parseFloat(part2);
+      return NumericRangeQuery.newFloatRange(field, precisionStep, min, max, inclusive, inclusive);
+    } else if (analyzer instanceof IntegerAnalyzer) {
+      IntegerAnalyzer a = (IntegerAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      int radix = a.getRadix();
+      int min = Integer.parseInt(part1, radix);
+      int max = Integer.parseInt(part2, radix);
+      return NumericRangeQuery.newIntRange(field, precisionStep, min, max, inclusive, inclusive);
+    }
+    return null;
+  }
+
   public boolean isFullTextField(String fieldName) {
     Boolean b = _fullTextFields.get(fieldName);
     if (b != null) {
@@ -271,7 +320,7 @@ public class BlurAnalyzer extends Analyzer {
   }
 
   private static void load(Map<String, Analyzer> analyzers, Map<String, ColumnFamilyDefinition>
familyDefinitions, Map<String, Boolean> fullTextFields,
-      Map<String, Set<String>> subIndexNameLookups, Map<String, Store>
storeMap, Map<String, Boolean> fullTextColumnFamilies) {
+      Map<String, Set<String>> subIndexNameLookups, Map<String, Store>
storeMap, Map<String, Boolean> fullTextColumnFamilies, Map<String, TYPE> typeLookup)
{
     if (familyDefinitions != null) {
       for (String family : familyDefinitions.keySet()) {
         ColumnFamilyDefinition familyDefinition = familyDefinitions.get(family);
@@ -279,33 +328,33 @@ public class BlurAnalyzer extends Analyzer {
         if (defaultDefinition != null) {
           fullTextColumnFamilies.put(family, defaultDefinition.isFullTextIndex());
         }
-        load(family, familyDefinition, analyzers, fullTextFields, subIndexNameLookups, storeMap);
+        load(family, familyDefinition, analyzers, fullTextFields, subIndexNameLookups, storeMap,
typeLookup);
       }
     }
   }
 
   private static void load(String family, ColumnFamilyDefinition familyDefinition, Map<String,
Analyzer> analyzers, Map<String, Boolean> fullTextFields,
-      Map<String, Set<String>> subIndexNameLookups, Map<String, Store>
storeMap) {
+      Map<String, Set<String>> subIndexNameLookups, Map<String, Store>
storeMap, Map<String, TYPE> typeLookup) {
     Map<String, ColumnDefinition> columnDefinitions = familyDefinition.getColumnDefinitions();
     if (columnDefinitions != null) {
       for (String column : columnDefinitions.keySet()) {
         ColumnDefinition columnDefinition = columnDefinitions.get(column);
-        load(family, familyDefinition, column, columnDefinition, analyzers, fullTextFields,
subIndexNameLookups, storeMap);
+        load(family, familyDefinition, column, columnDefinition, analyzers, fullTextFields,
subIndexNameLookups, storeMap, typeLookup);
       }
     }
   }
 
   private static void load(String family, ColumnFamilyDefinition familyDefinition, String
column, ColumnDefinition columnDefinition, Map<String, Analyzer> analyzers,
-      Map<String, Boolean> fullTextFields, Map<String, Set<String>> subIndexNameLookups,
Map<String, Store> storeMap) {
+      Map<String, Boolean> fullTextFields, Map<String, Set<String>> subIndexNameLookups,
Map<String, Store> storeMap, Map<String, TYPE> typeLookup) {
     Map<String, AlternateColumnDefinition> alternateColumnDefinitions = columnDefinition.getAlternateColumnDefinitions();
     if (alternateColumnDefinitions != null) {
       for (String subColumn : alternateColumnDefinitions.keySet()) {
         AlternateColumnDefinition alternateColumnDefinition = alternateColumnDefinitions.get(subColumn);
-        load(family, familyDefinition, column, columnDefinition, subColumn, alternateColumnDefinition,
analyzers, subIndexNameLookups, storeMap);
+        load(family, familyDefinition, column, columnDefinition, subColumn, alternateColumnDefinition,
analyzers, subIndexNameLookups, storeMap, typeLookup);
       }
     }
     String fieldName = family + "." + column;
-    Analyzer analyzer = getAnalyzerByClassName(columnDefinition.getAnalyzerClassName(), aliases);
+    Analyzer analyzer = getAnalyzerByClassName(columnDefinition.getAnalyzerClassName(), aliases,
fieldName, typeLookup);
     analyzers.put(fieldName, analyzer);
     if (columnDefinition.isFullTextIndex()) {
       fullTextFields.put(fieldName, Boolean.TRUE);
@@ -315,9 +364,10 @@ public class BlurAnalyzer extends Analyzer {
   }
 
   private static void load(String family, ColumnFamilyDefinition familyDefinition, String
column, ColumnDefinition columnDefinition, String subColumn,
-      AlternateColumnDefinition alternateColumnDefinition, Map<String, Analyzer> analyzers,
Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap)
{
+      AlternateColumnDefinition alternateColumnDefinition, Map<String, Analyzer> analyzers,
Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap,
+      Map<String, TYPE> typeLookup) {
     String fieldName = family + "." + column + "." + subColumn;
-    Analyzer analyzer = getAnalyzerByClassName(alternateColumnDefinition.getAnalyzerClassName(),
aliases);
+    Analyzer analyzer = getAnalyzerByClassName(alternateColumnDefinition.getAnalyzerClassName(),
aliases, fieldName, typeLookup);
     analyzers.put(fieldName, analyzer);
     putStore(fieldName, Store.NO, storeMap);
     addSubField(fieldName, subIndexNameLookups);
@@ -328,8 +378,8 @@ public class BlurAnalyzer extends Analyzer {
   }
 
   @SuppressWarnings("unchecked")
-  private static Analyzer getAnalyzerByClassName(String className, Map<String, Class<?
extends Analyzer>> aliases) {
-    Analyzer type = getType(className);
+  private static Analyzer getAnalyzerByClassName(String className, Map<String, Class<?
extends Analyzer>> aliases, String fieldName, Map<String, TYPE> typeLookup) {
+    Analyzer type = getType(className, fieldName, typeLookup);
     if (type != null) {
       return type;
     }
@@ -349,7 +399,7 @@ public class BlurAnalyzer extends Analyzer {
     }
   }
 
-  private static Analyzer getType(String typeStr) {
+  private static Analyzer getType(String typeStr, String fieldName, Map<String, TYPE>
typeLookup) {
     if (typeStr == null) {
       return null;
     }
@@ -359,7 +409,10 @@ public class BlurAnalyzer extends Analyzer {
     if (!typeNameCache.contains(typeUpper)) {
       return null;
     }
-    TYPES t = TYPES.valueOf(typeUpper);
+    TYPE t = TYPE.valueOf(typeUpper);
+    if (fieldName != null) {
+      typeLookup.put(fieldName, t);
+    }
     switch (t) {
     case LONG:
       LongAnalyzer longAnalyzer = new LongAnalyzer();
@@ -407,4 +460,5 @@ public class BlurAnalyzer extends Analyzer {
     }
     set.add(name);
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
index 124229b..296cc0b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
@@ -16,7 +16,6 @@ package org.apache.blur.lucene.search;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.utils.BlurConstants.SEP;
 import static org.apache.blur.utils.BlurConstants.SUPER;
 
 import java.util.HashMap;
@@ -24,8 +23,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.BlurAnalyzer.TYPE;
 import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.queryParser.QueryParser;
@@ -39,21 +39,22 @@ import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Version;
 
-
 public class SuperParser extends QueryParser {
 
-  private Map<Query, String> fieldNames = new HashMap<Query, String>();
-  private boolean superSearch = true;
-  private Filter queryFilter;
+  private final Map<Query, String> fieldNames = new HashMap<Query, String>();
+  private final boolean superSearch;
+  private final Filter queryFilter;
   private final ScoreType scoreType;
+  private final BlurAnalyzer blurAnalyzer;
 
-  public SuperParser(Version matchVersion, Analyzer a, boolean superSearch, Filter queryFilter,
ScoreType scoreType) {
+  public SuperParser(Version matchVersion, BlurAnalyzer a, boolean superSearch, Filter queryFilter,
ScoreType scoreType) {
     super(matchVersion, SUPER, a);
     this.setAutoGeneratePhraseQueries(true);
     this.setAllowLeadingWildcard(true);
     this.superSearch = superSearch;
     this.queryFilter = queryFilter;
     this.scoreType = scoreType;
+    this.blurAnalyzer = a;
   }
 
   @Override
@@ -63,6 +64,11 @@ public class SuperParser extends QueryParser {
 
   @Override
   protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) {
+    String field = term.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type + "] which does
not support fuzzy queries.");
+    }
     return addField(super.newFuzzyQuery(term, minimumSimilarity, prefixLength), term.field());
   }
 
@@ -101,17 +107,31 @@ public class SuperParser extends QueryParser {
 
   @Override
   protected Query newPrefixQuery(Term prefix) {
-    return addField(super.newPrefixQuery(prefix), prefix.field());
+    String field = prefix.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type + "] which does
not support prefix queries.");
+    }
+    return addField(super.newPrefixQuery(prefix), field);
   }
 
   @Override
   protected Query newRangeQuery(String field, String part1, String part2, boolean inclusive)
{
+    Query q = blurAnalyzer.getNewRangeQuery(field, part1, part2, inclusive);
+    if (q != null) {
+      return addField(q, field);
+    }
     return addField(super.newRangeQuery(field, part1, part2, inclusive), field);
   }
 
   @Override
   protected Query newTermQuery(Term term) {
-    return addField(super.newTermQuery(term), term.field());
+    String field = term.field();
+    Query q = blurAnalyzer.getNewRangeQuery(field, term.text(), term.text(), true);
+    if (q != null) {
+      return addField(q, field);
+    }
+    return addField(super.newTermQuery(term), field);
   }
 
   @Override
@@ -119,6 +139,11 @@ public class SuperParser extends QueryParser {
     if (SUPER.equals(t.field()) && "*".equals(t.text())) {
       return new MatchAllDocsQuery();
     }
+    String field = t.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type + "] which does
not support wildcard queries.");
+    }
     return addField(super.newWildcardQuery(t), t.field());
   }
 
@@ -128,15 +153,22 @@ public class SuperParser extends QueryParser {
     }
     if (query instanceof BooleanQuery) {
       BooleanQuery booleanQuery = (BooleanQuery) query;
-      if (isSameGroupName(booleanQuery)) {
-        return newSuperQuery(query);
-      } else {
-        List<BooleanClause> clauses = booleanQuery.clauses();
-        for (BooleanClause clause : clauses) {
-          clause.setQuery(reprocess(clause.getQuery()));
-        }
-        return booleanQuery;
+      List<BooleanClause> clauses = booleanQuery.clauses();
+      for (BooleanClause bc : clauses) {
+        Query q = bc.getQuery();
+        bc.setQuery(newSuperQuery(q));
       }
+      return booleanQuery;
+
+      // if (isSameGroupName(booleanQuery)) {
+      // return newSuperQuery(query);
+      // } else {
+      // List<BooleanClause> clauses = booleanQuery.clauses();
+      // for (BooleanClause clause : clauses) {
+      // clause.setQuery(reprocess(clause.getQuery()));
+      // }
+      // return booleanQuery;
+      // }
     } else {
       return newSuperQuery(query);
     }
@@ -153,56 +185,56 @@ public class SuperParser extends QueryParser {
     return new FilteredQuery(query, queryFilter);
   }
 
-  private boolean isSameGroupName(BooleanQuery booleanQuery) {
-    String groupName = findFirstGroupName(booleanQuery);
-    if (groupName == null) {
-      return false;
-    }
-    return isSameGroupName(booleanQuery, groupName);
-  }
-
-  private boolean isSameGroupName(Query query, String groupName) {
-    if (query instanceof BooleanQuery) {
-      BooleanQuery booleanQuery = (BooleanQuery) query;
-      for (BooleanClause clause : booleanQuery.clauses()) {
-        if (!isSameGroupName(clause.getQuery(), groupName)) {
-          return false;
-        }
-      }
-      return true;
-    } else {
-      String fieldName = fieldNames.get(query);
-      String currentGroupName = getGroupName(fieldName);
-      if (groupName.equals(currentGroupName)) {
-        return true;
-      }
-      return false;
-    }
-  }
-
-  private String getGroupName(String fieldName) {
-    if (fieldName == null) {
-      return null;
-    }
-    int index = fieldName.indexOf(SEP);
-    if (index < 0) {
-      return null;
-    }
-    return fieldName.substring(0, index);
-  }
-
-  private String findFirstGroupName(Query query) {
-    if (query instanceof BooleanQuery) {
-      BooleanQuery booleanQuery = (BooleanQuery) query;
-      for (BooleanClause clause : booleanQuery.clauses()) {
-        return findFirstGroupName(clause.getQuery());
-      }
-      return null;
-    } else {
-      String fieldName = fieldNames.get(query);
-      return getGroupName(fieldName);
-    }
-  }
+//  private boolean isSameGroupName(BooleanQuery booleanQuery) {
+//    String groupName = findFirstGroupName(booleanQuery);
+//    if (groupName == null) {
+//      return false;
+//    }
+//    return isSameGroupName(booleanQuery, groupName);
+//  }
+//
+//  private boolean isSameGroupName(Query query, String groupName) {
+//    if (query instanceof BooleanQuery) {
+//      BooleanQuery booleanQuery = (BooleanQuery) query;
+//      for (BooleanClause clause : booleanQuery.clauses()) {
+//        if (!isSameGroupName(clause.getQuery(), groupName)) {
+//          return false;
+//        }
+//      }
+//      return true;
+//    } else {
+//      String fieldName = fieldNames.get(query);
+//      String currentGroupName = getGroupName(fieldName);
+//      if (groupName.equals(currentGroupName)) {
+//        return true;
+//      }
+//      return false;
+//    }
+//  }
+//
+//  private String getGroupName(String fieldName) {
+//    if (fieldName == null) {
+//      return null;
+//    }
+//    int index = fieldName.indexOf(SEP);
+//    if (index < 0) {
+//      return null;
+//    }
+//    return fieldName.substring(0, index);
+//  }
+//
+//  private String findFirstGroupName(Query query) {
+//    if (query instanceof BooleanQuery) {
+//      BooleanQuery booleanQuery = (BooleanQuery) query;
+//      for (BooleanClause clause : booleanQuery.clauses()) {
+//        return findFirstGroupName(clause.getQuery());
+//      }
+//      return null;
+//    } else {
+//      String fieldName = fieldNames.get(query);
+//      return getGroupName(fieldName);
+//    }
+//  }
 
   private Query addField(Query q, String field) {
     fieldNames.put(q, field);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
index ef24626..bd83748 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
@@ -264,4 +264,8 @@ public class SuperQuery extends AbstractWrapperQuery {
       return doc == NO_MORE_DOCS ? true : false;
     }
   }
+
+  public Query getQuery() {
+    return _query;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index bc898b4..2bbb646 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -44,6 +44,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLongArray;
 
+import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -77,13 +78,12 @@ import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurExecutorCompletionService;
-import org.apache.blur.utils.ForkJoin;
-import org.apache.blur.utils.TermDocIterable;
 import org.apache.blur.utils.BlurExecutorCompletionService.Cancel;
+import org.apache.blur.utils.ForkJoin;
 import org.apache.blur.utils.ForkJoin.Merger;
 import org.apache.blur.utils.ForkJoin.ParallelCall;
+import org.apache.blur.utils.TermDocIterable;
 import org.apache.hadoop.io.BytesWritable;
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.document.FieldSelectorResult;
@@ -297,7 +297,7 @@ public class IndexManager {
         LOG.error("Unknown error while trying to fetch index readers.", e);
         throw new BException(e.getMessage(), e);
       }
-      Analyzer analyzer = _indexServer.getAnalyzer(table);
+      BlurAnalyzer analyzer = _indexServer.getAnalyzer(table);
       ParallelCall<Entry<String, BlurIndex>, BlurResultIterable> call;
       if (isSimpleQuery(blurQuery)) {
         SimpleQuery simpleQuery = blurQuery.simpleQuery;
@@ -345,14 +345,14 @@ public class IndexManager {
     return false;
   }
 
-  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts,
Analyzer analyzer) throws ParseException {
+  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts,
BlurAnalyzer analyzer) throws ParseException {
     if (blurQuery.facets == null) {
       return userQuery;
     }
     return new FacetQuery(userQuery, getFacetQueries(blurQuery, analyzer), counts);
   }
 
-  private Query[] getFacetQueries(BlurQuery blurQuery, Analyzer analyzer) throws ParseException
{
+  private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer) throws ParseException
{
     int size = blurQuery.facets.size();
     Query[] queries = new Query[size];
     for (int i = 0; i < size; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java b/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
index dc712be..3c4d92c 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
@@ -18,20 +18,19 @@ package org.apache.blur.manager;
  */
 import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
 
+import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.lucene.search.SuperParser;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryWrapperFilter;
 
-
 public class QueryParserUtil {
 
-  public static Query parseQuery(String query, boolean superQueryOn, Analyzer analyzer, Filter
postFilter, Filter preFilter, ScoreType scoreType) throws ParseException {
+  public static Query parseQuery(String query, boolean superQueryOn, BlurAnalyzer analyzer,
Filter postFilter, Filter preFilter, ScoreType scoreType) throws ParseException {
     Query result = new SuperParser(LUCENE_VERSION, analyzer, superQueryOn, preFilter, scoreType).parse(query);
     if (postFilter == null) {
       return result;
@@ -39,7 +38,7 @@ public class QueryParserUtil {
     return new FilteredQuery(result, postFilter);
   }
 
-  public static Filter parseFilter(String table, String filterStr, boolean superQueryOn,
Analyzer analyzer, BlurFilterCache filterCache) throws ParseException, BlurException {
+  public static Filter parseFilter(String table, String filterStr, boolean superQueryOn,
BlurAnalyzer analyzer, BlurFilterCache filterCache) throws ParseException, BlurException {
     if (filterStr == null) {
       return null;
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
b/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
new file mode 100644
index 0000000..e70f146
--- /dev/null
+++ b/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
@@ -0,0 +1,219 @@
+package org.apache.blur.lucene.search;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.List;
+
+import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.blur.thrift.generated.ScoreType;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.Version;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SuperParserTest {
+
+  private BlurAnalyzer analyzer;
+
+  @Before
+  public void setup() {
+    AnalyzerDefinition ad = new AnalyzerDefinition();
+    ad.setDefaultDefinition(new ColumnDefinition(StandardAnalyzer.class.getName(), true,
null));
+    ColumnFamilyDefinition cfDef = new ColumnFamilyDefinition();
+    cfDef.putToColumnDefinitions("id_l", new ColumnDefinition("long", false, null));
+    cfDef.putToColumnDefinitions("id_d", new ColumnDefinition("double", false, null));
+    cfDef.putToColumnDefinitions("id_f", new ColumnDefinition("float", false, null));
+    cfDef.putToColumnDefinitions("id_i", new ColumnDefinition("integer", false, null));
+    ad.putToColumnFamilyDefinitions("a", cfDef);
+    analyzer = new BlurAnalyzer(ad);
+  }
+
+  @Test
+  public void test1() throws ParseException {
+    Query q = parseSq("(a.b:cool) (+a.c:cool a.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc_m(tq("a.c", "cool")), bc(tq("a.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test2() throws ParseException {
+    Query q = parseSq("(a.b:cool) (a.c:cool a.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("a.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test3() throws ParseException {
+    Query q = parseSq("a.b:cool (a.c:cool a.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("a.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test4() throws ParseException {
+    Query q = parseSq("a.b:cool a.c:cool a.b:cool");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(tq("a.c", "cool"))), bc(sq(tq("a.b",
"cool")))), q);
+  }
+
+  @Test
+  public void test5() throws ParseException {
+    Query q = parseSq("(a.b:cool) (+a.c:cool c.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc_m(tq("a.c", "cool")), bc(tq("c.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test6() throws ParseException {
+    Query q = parseSq("(a.b:cool) (a.c:cool c.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("c.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test7() throws ParseException {
+    Query q = parseSq("a.b:cool (a.c:cool c.b:cool)");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("c.b",
"cool")))))), q);
+  }
+
+  @Test
+  public void test8() throws ParseException {
+    Query q = parseSq("a.b:cool a.c:cool c.b:cool");
+    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(tq("a.c", "cool"))), bc(sq(tq("c.b",
"cool")))), q);
+  }
+
+  @Test
+  public void test9() throws ParseException {
+    Query q = parseSq("a.id_l:[0 TO 2]");
+    assertQuery(sq(rq_i("a.id_l", 0L, 2L)), q);
+  }
+  
+  @Test
+  public void test10() throws ParseException {
+    Query q = parseSq("a.id_d:[0 TO 2]");
+    assertQuery(sq(rq_i("a.id_d", 0.0D, 2.0D)), q);
+  }
+  
+  @Test
+  public void test11() throws ParseException {
+    Query q = parseSq("a.id_f:[0 TO 2]");
+    assertQuery(sq(rq_i("a.id_f", 0.0F, 2.0F)), q);
+  }
+  
+  @Test
+  public void test12() throws ParseException {
+    Query q = parseSq("a.id_i:[0 TO 2]");
+    assertQuery(sq(rq_i("a.id_i", 0, 2)), q);
+  }
+
+  public static BooleanClause bc_m(Query q) {
+    return new BooleanClause(q, Occur.MUST);
+  }
+
+  public static BooleanClause bc_n(Query q) {
+    return new BooleanClause(q, Occur.MUST_NOT);
+  }
+
+  public static BooleanClause bc(Query q) {
+    return new BooleanClause(q, Occur.SHOULD);
+  }
+
+  public static void assertQuery(Query expected, Query actual) {
+    System.out.println(expected);
+    System.out.println(actual);
+    assertEqualsQuery(expected, actual);
+  }
+
+  public static void assertEqualsQuery(Query expected, Query actual) {
+    assertEquals(expected.getClass(), actual.getClass());
+    if (expected instanceof BooleanQuery) {
+      assertEqualsBooleanQuery((BooleanQuery) expected, (BooleanQuery) actual);
+    } else if (expected instanceof SuperQuery) {
+      assertEqualsSuperQuery((SuperQuery) expected, (SuperQuery) actual);
+    } else if (expected instanceof TermQuery) {
+      assertEqualsTermQuery((TermQuery) expected, (TermQuery) actual);
+    } else {
+      fail("Type [" + expected.getClass() + "] not supported");
+    }
+  }
+
+  public static void assertEqualsTermQuery(TermQuery expected, TermQuery actual) {
+    Term term1 = expected.getTerm();
+    Term term2 = actual.getTerm();
+    assertEquals(term1, term2);
+  }
+
+  public static void assertEqualsSuperQuery(SuperQuery expected, SuperQuery actual) {
+    assertEquals(expected.getQuery(), actual.getQuery());
+  }
+
+  public static void assertEqualsBooleanQuery(BooleanQuery expected, BooleanQuery actual)
{
+    List<BooleanClause> clauses1 = expected.clauses();
+    List<BooleanClause> clauses2 = actual.clauses();
+    assertEqualsBooleanClause(clauses1, clauses2);
+  }
+
+  public static void assertEqualsBooleanClause(List<BooleanClause> clauses1, List<BooleanClause>
clauses2) {
+    if (clauses1 == null && clauses2 == null) {
+      return;
+    }
+    if (clauses1 == null || clauses2 == null) {
+      fail();
+    }
+    if (clauses1.size() != clauses2.size()) {
+      fail();
+    }
+    int size = clauses1.size();
+    for (int i = 0; i < size; i++) {
+      assertEqualsBooleanClause(clauses1.get(i), clauses2.get(i));
+    }
+  }
+
+  public static void assertEqualsBooleanClause(BooleanClause booleanClause1, BooleanClause
booleanClause2) {
+    assertEquals(booleanClause1.getOccur(), booleanClause2.getOccur());
+    assertEqualsQuery(booleanClause1.getQuery(), booleanClause2.getQuery());
+  }
+  
+  private Query rq_i(String field, float min, float max) {
+    return NumericRangeQuery.newFloatRange(field, min, max, true, true);
+  }
+  
+  private Query rq_i(String field, int min, int max) {
+    return NumericRangeQuery.newIntRange(field, min, max, true, true);
+  }
+  
+  private Query rq_i(String field, double min, double max) {
+    return NumericRangeQuery.newDoubleRange(field, min, max, true, true);
+  }
+
+  private Query rq_i(String field, long min, long max) {
+    return NumericRangeQuery.newLongRange(field, min, max, true, true);
+  }
+
+  private BooleanQuery bq(BooleanClause... bcs) {
+    BooleanQuery bq = new BooleanQuery();
+    for (BooleanClause bc : bcs) {
+      bq.add(bc);
+    }
+    return bq;
+  }
+
+  private SuperQuery sq(Query q) {
+    return new SuperQuery(q, ScoreType.SUPER);
+  }
+
+  private TermQuery tq(String field, String text) {
+    return new TermQuery(new Term(field, text));
+  }
+
+  private Query parseSq(String qstr) throws ParseException {
+    SuperParser parser = new SuperParser(Version.LUCENE_36, analyzer, true, null, ScoreType.SUPER);
+    return parser.parse(qstr);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/af8229b4/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
index 1045c7a..132112d 100644
--- a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
@@ -84,7 +84,7 @@ public class RandomSuperQueryTest {
     IndexSearcher searcher = new IndexSearcher(reader);
     long s = System.currentTimeMillis();
     for (String str : sampler) {
-      Query query = new SuperParser(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION),
true, filter, ScoreType.AGGREGATE).parse(str);
+      Query query = new SuperParser(LUCENE_VERSION, new BlurAnalyzer(), true, filter, ScoreType.AGGREGATE).parse(str);
       TopDocs topDocs = searcher.search(query, 10);
       assertTrue("seed [" + seed + "] {" + query + "} {" + s + "}", topDocs.totalHits >
0);
     }


Mime
View raw message