incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/6] BLUR-ID-64-changed-parser
Date Thu, 11 Apr 2013 03:10:50 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java b/src/blur-query/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
new file mode 100644
index 0000000..fb04888
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
@@ -0,0 +1,86 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+public final class FloatAnalyzer extends Analyzer {
+
+  public static int PRECISION_STEP_DEFAULT = NumericUtils.PRECISION_STEP_DEFAULT;
+  private int precisionStep;
+
+  public FloatAnalyzer() {
+    this(PRECISION_STEP_DEFAULT);
+  }
+
+  public FloatAnalyzer(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+
+  public int getPrecisionStep() {
+    return precisionStep;
+  }
+
+  public void setPrecisionStep(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+
+  private float toFloat(Reader reader) throws IOException {
+    StringBuilder builder = new StringBuilder(20);
+    int read;
+    while ((read = reader.read()) != -1) {
+      builder.append((char) read);
+    }
+    return Float.parseFloat(builder.toString());
+  }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final float value;
+    try {
+      value = toFloat(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setFloatValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setFloatValue(toFloat(reader));
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java b/src/blur-query/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
new file mode 100644
index 0000000..f63330e
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
@@ -0,0 +1,101 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+public final class IntegerAnalyzer extends Analyzer {
+
+  public static int PRECISION_STEP_DEFAULT = NumericUtils.PRECISION_STEP_DEFAULT;
+  public static int RADIX_DEFAULT = 10;
+  private int radix = 10;
+  private int precisionStep;
+
+  public IntegerAnalyzer() {
+    this(PRECISION_STEP_DEFAULT, RADIX_DEFAULT);
+  }
+
+  public IntegerAnalyzer(int precisionStep) {
+    this(precisionStep, RADIX_DEFAULT);
+  }
+
+  public IntegerAnalyzer(int precisionStep, int radix) {
+    this.precisionStep = precisionStep;
+    this.radix = radix;
+  }
+
+  public int getRadix() {
+    return radix;
+  }
+
+  public void setRadix(int radix) {
+    this.radix = radix;
+  }
+
+  public int getPrecisionStep() {
+    return precisionStep;
+  }
+
+  public void setPrecisionStep(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+
+  private int toInteger(Reader reader) throws IOException {
+    StringBuilder builder = new StringBuilder(20);
+    int read;
+    while ((read = reader.read()) != -1) {
+      builder.append((char) read);
+    }
+    return Integer.parseInt(builder.toString(), radix);
+  }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final int value;
+    try {
+      value = toInteger(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setIntValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setIntValue(toInteger(reader));
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/analysis/LongAnalyzer.java b/src/blur-query/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
new file mode 100644
index 0000000..b8d7a54
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
@@ -0,0 +1,101 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+public final class LongAnalyzer extends Analyzer {
+
+  public static int PRECISION_STEP_DEFAULT = NumericUtils.PRECISION_STEP_DEFAULT;
+  public static int RADIX_DEFAULT = 10;
+  private int radix = 10;
+  private int precisionStep;
+
+  public LongAnalyzer() {
+    this(PRECISION_STEP_DEFAULT, RADIX_DEFAULT);
+  }
+
+  public LongAnalyzer(int precisionStep) {
+    this(precisionStep, RADIX_DEFAULT);
+  }
+
+  public LongAnalyzer(int precisionStep, int radix) {
+    this.precisionStep = precisionStep;
+    this.radix = radix;
+  }
+
+  public int getRadix() {
+    return radix;
+  }
+
+  public void setRadix(int radix) {
+    this.radix = radix;
+  }
+
+  public int getPrecisionStep() {
+    return precisionStep;
+  }
+
+  public void setPrecisionStep(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+
+  private long toLong(Reader reader) throws IOException {
+    StringBuilder builder = new StringBuilder(20);
+    int read;
+    while ((read = reader.read()) != -1) {
+      builder.append((char) read);
+    }
+    return Long.parseLong(builder.toString(), radix);
+  }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final long value;
+    try {
+      value = toLong(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setLongValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setLongValue(toLong(reader));
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
new file mode 100644
index 0000000..8c8b945
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
@@ -0,0 +1,61 @@
+package org.apache.blur.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
+import org.apache.lucene.util.BytesRef;
+
+public class FairSimilarity extends TFIDFSimilarity {
+
+  @Override
+  public float coord(int overlap, int maxOverlap) {
+    return 1;
+  }
+
+  @Override
+  public float idf(long docFreq, long numDocs) {
+    return 1;
+  }
+
+  @Override
+  public float queryNorm(float sumOfSquaredWeights) {
+    return 1;
+  }
+
+  @Override
+  public float sloppyFreq(int distance) {
+    return 1;
+  }
+
+  @Override
+  public float tf(float freq) {
+    return 1;
+  }
+
+  @Override
+  public float scorePayload(int doc, int start, int end, BytesRef payload) {
+    return 1;
+  }
+
+  @Override
+  public float lengthNorm(FieldInvertState fieldInvertState) {
+    throw new RuntimeException("not sure");
+//    return 0;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
new file mode 100644
index 0000000..586b68a
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
@@ -0,0 +1,227 @@
+package org.apache.blur.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollector;
+
+/**
+ * The {@link IterablePaging} class allows for easy paging through lucene hits.
+ */
+public class IterablePaging implements Iterable<ScoreDoc> {
+
+  private final IndexSearcher searcher;
+  private final Query query;
+  private final AtomicBoolean running;
+  private final int numHitsToCollect;
+
+  private TotalHitsRef totalHitsRef;
+  private ProgressRef progressRef;
+  private int skipTo;
+  private int gather = -1;
+
+  public IterablePaging(AtomicBoolean running, IndexSearcher searcher, Query query,
+      int numHitsToCollect, TotalHitsRef totalHitsRef, ProgressRef progressRef) throws IOException {
+    this.running = running;
+    this.query = searcher.rewrite(query);
+    this.searcher = searcher;
+    this.numHitsToCollect = numHitsToCollect;
+    this.totalHitsRef = totalHitsRef == null ? new TotalHitsRef() : totalHitsRef;
+    this.progressRef = progressRef == null ? new ProgressRef() : progressRef;
+
+  }
+
+  public static class TotalHitsRef {
+    // This is an atomic integer because more than likely if there is
+    // any status sent to the user, it will be done in another thread.
+    protected AtomicInteger totalHits = new AtomicInteger(0);
+
+    public int totalHits() {
+      return totalHits.get();
+    }
+  }
+
+  public static class ProgressRef {
+    // These are atomic integers because more than likely if there is
+    // any status sent to the user, it will be done in another thread.
+    protected AtomicInteger skipTo = new AtomicInteger(0);
+    protected AtomicInteger currentHitPosition = new AtomicInteger(0);
+    protected AtomicInteger searchesPerformed = new AtomicInteger(0);
+    protected AtomicLong queryTime = new AtomicLong(0);
+
+    public int skipTo() {
+      return skipTo.get();
+    }
+
+    public int currentHitPosition() {
+      return currentHitPosition.get();
+    }
+
+    public int searchesPerformed() {
+      return searchesPerformed.get();
+    }
+
+    public long queryTime() {
+      return queryTime.get();
+    }
+  }
+
+  /**
+   * Gets the total hits of the search.
+   * 
+   * @return the total hits.
+   */
+  public int getTotalHits() {
+    return totalHitsRef.totalHits();
+  }
+
+  /**
+   * Allows for gathering of the total hits of this search.
+   * 
+   * @param ref
+   *          {@link TotalHitsRef}.
+   * @return this.
+   */
+  public IterablePaging totalHits(TotalHitsRef ref) {
+    totalHitsRef = ref;
+    return this;
+  }
+
+  /**
+   * Skips the first x number of hits.
+   * 
+   * @param skipTo
+   *          the number hits to skip.
+   * @return this.
+   */
+  public IterablePaging skipTo(int skipTo) {
+    this.skipTo = skipTo;
+    return this;
+  }
+
+  /**
+   * Only gather up to x number of hits.
+   * 
+   * @param gather
+   *          the number of hits to gather.
+   * @return this.
+   */
+  public IterablePaging gather(int gather) {
+    this.gather = gather;
+    return this;
+  }
+
+  /**
+   * Allows for gathering the progress of the paging.
+   * 
+   * @param ref
+   *          the {@link ProgressRef}.
+   * @return this.
+   */
+  public IterablePaging progress(ProgressRef ref) {
+    this.progressRef = ref;
+    return this;
+  }
+
+  /**
+   * The {@link ScoreDoc} iterator.
+   */
+  @Override
+  public Iterator<ScoreDoc> iterator() {
+    return skipHits(new PagingIterator());
+  }
+
+  class PagingIterator implements Iterator<ScoreDoc> {
+    private ScoreDoc[] scoreDocs;
+    private int counter = 0;
+    private int offset = 0;
+    private int endPosition = gather == -1 ? Integer.MAX_VALUE : skipTo + gather;
+    private ScoreDoc lastScoreDoc;
+
+    PagingIterator() {
+      search();
+    }
+
+    void search() {
+      long s = System.currentTimeMillis();
+      progressRef.searchesPerformed.incrementAndGet();
+      try {
+        TopScoreDocCollector collector = TopScoreDocCollector.create(numHitsToCollect, lastScoreDoc, true);
+        StopExecutionCollector stopExecutionCollector = new StopExecutionCollector(collector, running);
+        searcher.search(query, stopExecutionCollector);
+        totalHitsRef.totalHits.set(collector.getTotalHits());
+        TopDocs topDocs = collector.topDocs();
+        scoreDocs = topDocs.scoreDocs;
+      } catch (IOException e) {
+        e.printStackTrace();
+        throw new RuntimeException(e);
+      }
+      if (scoreDocs.length > 0) {
+        lastScoreDoc = scoreDocs[scoreDocs.length - 1];
+      } else {
+        lastScoreDoc = null;
+      }
+      long e = System.currentTimeMillis();
+      progressRef.queryTime.addAndGet(e - s);
+    }
+
+    @Override
+    public boolean hasNext() {
+      return counter < totalHitsRef.totalHits() && counter < endPosition ? true : false;
+    }
+
+    @Override
+    public ScoreDoc next() {
+      if (isCurrentCollectorExhausted()) {
+        search();
+        offset = 0;
+      }
+      progressRef.currentHitPosition.set(counter);
+      counter++;
+      return scoreDocs[offset++];
+    }
+
+    private boolean isCurrentCollectorExhausted() {
+      return offset < scoreDocs.length ? false : true;
+    }
+
+    @Override
+    public void remove() {
+      throw new RuntimeException("read only");
+    }
+  }
+
+  private Iterator<ScoreDoc> skipHits(Iterator<ScoreDoc> iterator) {
+    progressRef.skipTo.set(skipTo);
+    for (int i = 0; i < skipTo && iterator.hasNext(); i++) {
+      // eats the hits, and moves the iterator to the desired skip to position.
+      progressRef.currentHitPosition.set(i);
+      iterator.next();
+    }
+    return iterator;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/PagingCollector.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
new file mode 100644
index 0000000..fec888c
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
@@ -0,0 +1,116 @@
+package org.apache.blur.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopDocsCollector;
+import org.apache.lucene.util.PriorityQueue;
+
+/**
+ * The {@link PagingCollector} allows for paging through lucene hits.
+ */
+public class PagingCollector extends TopDocsCollector<ScoreDoc> {
+
+  private ScoreDoc pqTop;
+  private int docBase;
+  private Scorer scorer;
+  private ScoreDoc previousPassLowest;
+  private int numHits;
+
+  public PagingCollector(int numHits) {
+    // creates an empty score doc so that i don't have to check for null
+    // each time.
+    this(numHits, new ScoreDoc(-1, Float.MAX_VALUE));
+  }
+
+  public PagingCollector(int numHits, ScoreDoc previousPassLowest) {
+    super(new HitQueue(numHits, true));
+    this.pqTop = pq.top();
+    this.numHits = numHits;
+    this.previousPassLowest = previousPassLowest;
+  }
+
+  @Override
+  public boolean acceptsDocsOutOfOrder() {
+    return true;
+  }
+
+  @Override
+  public void collect(int doc) throws IOException {
+    float score = scorer.score();
+    totalHits++;
+    doc += docBase;
+    if (score > previousPassLowest.score) {
+      // this hit was gathered on a previous page.
+      return;
+    } else if (score == previousPassLowest.score && doc <= previousPassLowest.doc) {
+      // if the scores are the same and the doc is less than or equal to the
+      // previous pass lowest hit doc then skip because this collector favors
+      // lower number documents.
+      return;
+    } else if (score < pqTop.score || (score == pqTop.score && doc > pqTop.doc)) {
+      return;
+    }
+    pqTop.doc = doc;
+    pqTop.score = score;
+    pqTop = pq.updateTop();
+  }
+
+  @Override
+  public void setNextReader(AtomicReaderContext context) throws IOException {
+    this.docBase = context.docBase;
+  }
+
+  @Override
+  public void setScorer(Scorer scorer) throws IOException {
+    this.scorer = scorer;
+  }
+
+  public ScoreDoc getLastScoreDoc(TopDocs topDocs) {
+    return topDocs.scoreDocs[(totalHits < numHits ? totalHits : numHits) - 1];
+  }
+
+  public ScoreDoc getLastScoreDoc(ScoreDoc[] scoreDocs) {
+    return scoreDocs[(totalHits < numHits ? totalHits : numHits) - 1];
+  }
+
+  public static class HitQueue extends PriorityQueue<ScoreDoc> {
+
+    HitQueue(int size, boolean prePopulate) {
+      super(size, prePopulate);
+    }
+
+    @Override
+    protected ScoreDoc getSentinelObject() {
+      return new ScoreDoc(Integer.MAX_VALUE, Float.NEGATIVE_INFINITY);
+    }
+
+    @Override
+    protected final boolean lessThan(ScoreDoc hitA, ScoreDoc hitB) {
+      if (hitA.score == hitB.score)
+        return hitA.doc > hitB.doc;
+      else
+        return hitA.score < hitB.score;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/ScoreType.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/ScoreType.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/ScoreType.java
deleted file mode 100644
index f32c7f0..0000000
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/ScoreType.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.blur.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public enum ScoreType {
-  AGGREGATE, BEST, CONSTANT, SUPER
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
index 80a53fa..21488c3 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java
@@ -1,14 +1,43 @@
 package org.apache.blur.lucene.search;
 
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.lucene.analysis.Analyzer;
+import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.BlurAnalyzer.TYPE;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilteredQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
+import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Version;
 
@@ -18,20 +47,25 @@ public class SuperParser extends QueryParser {
   private static final String MUST_STRING = "+";
   private static final Pattern PATTERN = Pattern.compile("([-+]{0,1})\\s*?super\\s*?\\:\\s*?\\<(.*?)\\>");
   private static final Pattern CHECK = Pattern.compile("super\\s*?\\:\\s*?\\<");
-
-  private final Analyzer a;
-  private final String f;
+  private static final String SUPER = "super";
+  private final Map<Query, String> fieldNames = new HashMap<Query, String>();
+  private final boolean superSearch;
+  private final Filter queryFilter;
+  private final ScoreType scoreType;
+  private final BlurAnalyzer blurAnalyzer;
   private final Version matchVersion;
   private final Term defaultPrimeDocTerm;
-  private final ScoreType defaultScoreType;
 
-  public SuperParser(Version matchVersion, String f, Analyzer a, ScoreType defaultScoreType, Term defaultPrimeDocTerm) {
-    super(matchVersion, f, a);
+  public SuperParser(Version matchVersion, BlurAnalyzer a, boolean superSearch, Filter queryFilter, ScoreType scoreType, Term defaultPrimeDocTerm) {
+    super(matchVersion, "super", a);
     this.matchVersion = matchVersion;
-    this.f = f;
-    this.a = a;
+    this.setAutoGeneratePhraseQueries(true);
+    this.setAllowLeadingWildcard(true);
+    this.superSearch = superSearch;
+    this.queryFilter = queryFilter;
+    this.scoreType = scoreType;
+    this.blurAnalyzer = a;
     this.defaultPrimeDocTerm = defaultPrimeDocTerm;
-    this.defaultScoreType = defaultScoreType;
   }
 
   @Override
@@ -46,7 +80,9 @@ public class SuperParser extends QueryParser {
         String superQueryStr = matcher.group(i + 1);
         Matcher matcherCheck = CHECK.matcher(superQueryStr);
         if (matcherCheck.find()) {
-          throw new ParseException("Embedded super queries are not allowed [" + query + "].");
+          throw new ParseException(
+              "Embedded super queries are not allowed [" + query
+                  + "].");
         }
 
         if (booleanQuery == null) {
@@ -54,10 +90,10 @@ public class SuperParser extends QueryParser {
         }
 
         Occur occur = getOccur(occurString);
-        QueryParser parser = new QueryParser(matchVersion, f, a);
+        QueryParser parser = new QueryParser(matchVersion, SUPER, blurAnalyzer);
 
         Query superQuery = parser.parse(superQueryStr);
-        booleanQuery.add(new SuperQuery(superQuery, defaultScoreType, defaultPrimeDocTerm), occur);
+        booleanQuery.add(new SuperQuery(superQuery, scoreType, defaultPrimeDocTerm), occur);
       }
     }
     if (booleanQuery == null) {
@@ -76,4 +112,193 @@ public class SuperParser extends QueryParser {
     return Occur.SHOULD;
   }
 
+  @Override
+  protected Query newFuzzyQuery(Term term, float minimumSimilarity,
+      int prefixLength) {
+    String field = term.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type
+          + "] which does not support fuzzy queries.");
+    }
+    return addField(
+        super.newFuzzyQuery(term, minimumSimilarity, prefixLength),
+        term.field());
+  }
+
+  @Override
+  protected Query newMatchAllDocsQuery() {
+    return addField(super.newMatchAllDocsQuery(), UUID.randomUUID()
+        .toString());
+  }
+
+  @Override
+  protected MultiPhraseQuery newMultiPhraseQuery() {
+    return new MultiPhraseQuery() {
+
+      @Override
+      public void add(Term[] terms, int position) {
+        super.add(terms, position);
+        for (Term term : terms) {
+          addField(this, term.field());
+        }
+      }
+    };
+  }
+
+  @Override
+  protected PhraseQuery newPhraseQuery() {
+    return new PhraseQuery() {
+
+      @Override
+      public void add(Term term, int position) {
+        super.add(term, position);
+        addField(this, term.field());
+      }
+    };
+  }
+
+  @Override
+  protected Query newPrefixQuery(Term prefix) {
+    String field = prefix.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type
+          + "] which does not support prefix queries.");
+    }
+    return addField(super.newPrefixQuery(prefix), field);
+  }
+
+  @Override
+  protected Query newRangeQuery(String field, String part1, String part2,
+      boolean startInclusive, boolean endInclusive) {
+    Query q = blurAnalyzer.getNewRangeQuery(field, part1, part2,
+        startInclusive, endInclusive);
+    if (q != null) {
+      return addField(q, field);
+    }
+    return addField(super.newRangeQuery(field, part1, part2,
+        startInclusive, endInclusive), field);
+  }
+
+  @Override
+  protected Query newTermQuery(Term term) {
+    String field = term.field();
+    Query q = blurAnalyzer.getNewRangeQuery(field, term.text(),
+        term.text(), true, true);
+    if (q != null) {
+      return addField(q, field);
+    }
+    return addField(super.newTermQuery(term), field);
+  }
+
+  @Override
+  protected Query newWildcardQuery(Term t) {
+    if (SUPER.equals(t.field()) && "*".equals(t.text())) {
+      return new MatchAllDocsQuery();
+    }
+    String field = t.field();
+    TYPE type = blurAnalyzer.getTypeLookup(field);
+    if (type != TYPE.DEFAULT) {
+      throw new RuntimeException("Field [" + field + "] is type [" + type
+          + "] which does not support wildcard queries.");
+    }
+    return addField(super.newWildcardQuery(t), t.field());
+  }
+
+  private SuperQuery newSuperQuery(Query query) {
+    return new SuperQuery(wrapFilter(query), scoreType, defaultPrimeDocTerm);
+  }
+
+  private Query wrapFilter(Query query) {
+    if (queryFilter == null) {
+      return query;
+    }
+    return new FilteredQuery(query, queryFilter);
+  }
+
+  // private boolean isSameGroupName(BooleanQuery booleanQuery) {
+  // String groupName = findFirstGroupName(booleanQuery);
+  // if (groupName == null) {
+  // return false;
+  // }
+  // return isSameGroupName(booleanQuery, groupName);
+  // }
+  //
+  // private boolean isSameGroupName(Query query, String groupName) {
+  // if (query instanceof BooleanQuery) {
+  // BooleanQuery booleanQuery = (BooleanQuery) query;
+  // for (BooleanClause clause : booleanQuery.clauses()) {
+  // if (!isSameGroupName(clause.getQuery(), groupName)) {
+  // return false;
+  // }
+  // }
+  // return true;
+  // } else {
+  // String fieldName = fieldNames.get(query);
+  // String currentGroupName = getGroupName(fieldName);
+  // if (groupName.equals(currentGroupName)) {
+  // return true;
+  // }
+  // return false;
+  // }
+  // }
+  //
+  // private String getGroupName(String fieldName) {
+  // if (fieldName == null) {
+  // return null;
+  // }
+  // int index = fieldName.indexOf(SEP);
+  // if (index < 0) {
+  // return null;
+  // }
+  // return fieldName.substring(0, index);
+  // }
+  //
+  // private String findFirstGroupName(Query query) {
+  // if (query instanceof BooleanQuery) {
+  // BooleanQuery booleanQuery = (BooleanQuery) query;
+  // for (BooleanClause clause : booleanQuery.clauses()) {
+  // return findFirstGroupName(clause.getQuery());
+  // }
+  // return null;
+  // } else {
+  // String fieldName = fieldNames.get(query);
+  // return getGroupName(fieldName);
+  // }
+  // }
+  private Query reprocess(Query query) {
+    if (query == null || !isSuperSearch()) {
+      return wrapFilter(query);
+    }
+    if (query instanceof BooleanQuery) {
+      BooleanQuery booleanQuery = (BooleanQuery) query;
+      List<BooleanClause> clauses = booleanQuery.clauses();
+      for (BooleanClause bc : clauses) {
+        Query q = bc.getQuery();
+        bc.setQuery(newSuperQuery(q));
+      }
+      return booleanQuery;
+
+      // if (isSameGroupName(booleanQuery)) {
+      // return newSuperQuery(query);
+      // } else {
+      // List<BooleanClause> clauses = booleanQuery.clauses();
+      // for (BooleanClause clause : clauses) {
+      // clause.setQuery(reprocess(clause.getQuery()));
+      // }
+      // return booleanQuery;
+      // }
+    } else {
+      return newSuperQuery(query);
+    }
+  }
+  private Query addField(Query q, String field) {
+    fieldNames.put(q, field);
+    return q;
+  }
+
+  public boolean isSuperSearch() {
+    return superSearch;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
index 72be513..8bc01e3 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
@@ -28,6 +28,7 @@ import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.OpenBitSet;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/ProtoSerializer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/ProtoSerializer.java b/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/ProtoSerializer.java
index b90d8b2..37ac3a4 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/ProtoSerializer.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/ProtoSerializer.java
@@ -18,7 +18,7 @@ package org.apache.blur.lucene.serializer;
  */
 import java.io.IOException;
 
-import org.apache.blur.lucene.search.ScoreType;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.lucene.search.SuperQuery;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/SuperQueryWritable.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/SuperQueryWritable.java b/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/SuperQueryWritable.java
index d060728..b93206d 100644
--- a/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/SuperQueryWritable.java
+++ b/src/blur-query/src/main/java/org/apache/blur/lucene/serializer/SuperQueryWritable.java
@@ -4,7 +4,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
-import org.apache.blur.lucene.search.ScoreType;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.lucene.search.SuperQuery;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/util/BlurConstants.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/util/BlurConstants.java b/src/blur-query/src/main/java/org/apache/blur/util/BlurConstants.java
new file mode 100644
index 0000000..f86b577
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/util/BlurConstants.java
@@ -0,0 +1,28 @@
+package org.apache.blur.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+public class BlurConstants {
+
+  public static final String PRIME_DOC = "_prime_";
+  public static final String PRIME_DOC_VALUE = "true";
+  public static final String ROW_ID = "rowid";
+  public static final String RECORD_ID = "recordid";
+  public static final String SUPER = "super";
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java b/src/blur-query/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
new file mode 100644
index 0000000..f8d4ffb
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
@@ -0,0 +1,133 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.AlternateColumnDefinition;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.lucene.document.Field.Store;
+import org.junit.Test;
+
+public class BlurAnalyzerTest {
+
+  private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
+
+  @Test
+  public void testToAndFromJSONDef1() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
+    String json = analyzer.toJSON();
+    BlurAnalyzer analyzer2 = BlurAnalyzer.create(json);
+    assertEquals(analyzer.getAnalyzerDefinition(), analyzer2.getAnalyzerDefinition());
+  }
+
+  @Test
+  public void testStoringOfFieldDef1() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
+    assertEquals(Store.NO, analyzer.getStore("b.c.sub1"));
+    assertEquals(Store.YES, analyzer.getStore("b.c"));
+  }
+
+  @Test
+  public void testGetSubFieldsDef1() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
+    assertNull(analyzer.getSubIndexNames("b.d"));
+    Set<String> subIndexNames = analyzer.getSubIndexNames("b.c");
+    TreeSet<String> set = new TreeSet<String>();
+    set.add("b.c.sub1");
+    set.add("b.c.sub2");
+    assertEquals(set, subIndexNames);
+  }
+
+  @Test
+  public void testFullTextFieldsDef1() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
+    assertTrue(analyzer.isFullTextField("a.b"));
+    assertFalse(analyzer.isFullTextField("a.d"));
+  }
+
+  @Test
+  public void testToAndFromJSONDef2() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
+    String json = analyzer.toJSON();
+    BlurAnalyzer analyzer2 = BlurAnalyzer.create(json);
+    assertEquals(analyzer.getAnalyzerDefinition(), analyzer2.getAnalyzerDefinition());
+  }
+
+  @Test
+  public void testStoringOfFieldDef2() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
+    assertEquals(Store.YES, analyzer.getStore("a.b"));
+    assertEquals(Store.YES, analyzer.getStore("b.c"));
+  }
+
+  @Test
+  public void testGetSubFieldsDef2() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
+    assertNull(analyzer.getSubIndexNames("b.d"));
+  }
+
+  @Test
+  public void testFullTextFieldsDef2() throws IOException {
+    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
+    assertTrue(analyzer.isFullTextField("a.b"));
+    assertFalse(analyzer.isFullTextField("d.a"));
+  }
+
+  private AnalyzerDefinition getDef1() {
+
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, false, null)).setFullTextAnalyzerClassName(STANDARD);
+    Map<String, ColumnFamilyDefinition> columnFamilyDefinitions = new HashMap<String, ColumnFamilyDefinition>();
+
+    ColumnFamilyDefinition aColumnFamilyDefinition = new ColumnFamilyDefinition();
+
+    Map<String, ColumnDefinition> aColumnDefinitions = new HashMap<String, ColumnDefinition>();
+    aColumnDefinitions.put("b", new ColumnDefinition(STANDARD, true, null));
+    aColumnFamilyDefinition.setColumnDefinitions(aColumnDefinitions);
+    columnFamilyDefinitions.put("a", aColumnFamilyDefinition);
+
+    Map<String, ColumnDefinition> bColumnDefinitions = new HashMap<String, ColumnDefinition>();
+    Map<String, AlternateColumnDefinition> alternates = new HashMap<String, AlternateColumnDefinition>();
+    alternates.put("sub1", new AlternateColumnDefinition(STANDARD));
+    alternates.put("sub2", new AlternateColumnDefinition(STANDARD));
+    bColumnDefinitions.put("c", new ColumnDefinition(STANDARD, true, alternates));
+    ColumnFamilyDefinition bColumnFamilyDefinition = new ColumnFamilyDefinition();
+    bColumnFamilyDefinition.setColumnDefinitions(bColumnDefinitions);
+    columnFamilyDefinitions.put("b", bColumnFamilyDefinition);
+
+    analyzerDefinition.setColumnFamilyDefinitions(columnFamilyDefinitions);
+    return analyzerDefinition;
+  }
+
+  private AnalyzerDefinition getDef2() {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, false, null)).setFullTextAnalyzerClassName(STANDARD);
+    analyzerDefinition.putToColumnFamilyDefinitions("a", new ColumnFamilyDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, true, null)));
+    return analyzerDefinition;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java b/src/blur-query/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
new file mode 100644
index 0000000..7573025
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
@@ -0,0 +1,95 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.blur.index.IndexWriter;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class DoubleAnalyzerTest {
+
+  @Test
+  public void testLongAnalyzer() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("double", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStep() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("double,4", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer);
+  }
+
+  private void runTestString(Analyzer analyzer) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
+    Directory dir = new RAMDirectory();
+    IndexWriter indexWriter = new IndexWriter(dir, conf);
+    for (int i = 0; i < 1000; i++) {
+      Document document = new Document();
+      String value = Double.toString(i);
+      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
+    NumericRangeQuery<Double> query = NumericRangeQuery.newDoubleRange("test.test", 0.0, 2.0, true, true);
+    Query rewrite = searcher.rewrite(query);
+    TopDocs docs = searcher.search(rewrite, 100);
+    ScoreDoc[] scoreDocs = docs.scoreDocs;
+    assertEquals(3, docs.totalHits);
+    for (int i = 0; i < docs.totalHits; i++) {
+      Document document = searcher.doc(scoreDocs[i].doc);
+      assertTrue(Double.parseDouble(document.get("test.test")) < 3.0);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java b/src/blur-query/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
new file mode 100644
index 0000000..f989c90
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
@@ -0,0 +1,95 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.blur.index.IndexWriter;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class FloatAnalyzerTest {
+
+  @Test
+  public void testLongAnalyzer() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("float", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStep() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("float,4", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer);
+  }
+
+  private void runTestString(Analyzer analyzer) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
+    Directory dir = new RAMDirectory();
+    IndexWriter indexWriter = new IndexWriter(dir, conf);
+    for (int i = 0; i < 1000; i++) {
+      Document document = new Document();
+      String value = Float.toString(i);
+      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
+    NumericRangeQuery<Float> query = NumericRangeQuery.newFloatRange("test.test", 0.0F, 2.0F, true, true);
+    Query rewrite = searcher.rewrite(query);
+    TopDocs docs = searcher.search(rewrite, 100);
+    ScoreDoc[] scoreDocs = docs.scoreDocs;
+    assertEquals(3, docs.totalHits);
+    for (int i = 0; i < docs.totalHits; i++) {
+      Document document = searcher.doc(scoreDocs[i].doc);
+      assertTrue(Float.parseFloat(document.get("test.test")) < 3.0F);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java b/src/blur-query/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
new file mode 100644
index 0000000..6c28e6c
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
@@ -0,0 +1,106 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.blur.index.IndexWriter;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class IntegerAnalyzerTest {
+
+  @Test
+  public void testLongAnalyzer() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("integer", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 10);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStep() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("integer,4", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 10);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStepAndRadix() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("integer,4,16", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 16);
+  }
+
+  private void runTestString(Analyzer analyzer, int radix) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
+    Directory dir = new RAMDirectory();
+    IndexWriter indexWriter = new IndexWriter(dir, conf);
+    for (int i = 0; i < 1000; i++) {
+      Document document = new Document();
+      String value = Integer.toString(i, radix);
+      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
+    NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("test.test", 0, 2, true, true);
+    Query rewrite = searcher.rewrite(query);
+    TopDocs docs = searcher.search(rewrite, 100);
+    ScoreDoc[] scoreDocs = docs.scoreDocs;
+    assertEquals(3, docs.totalHits);
+    for (int i = 0; i < docs.totalHits; i++) {
+      Document document = searcher.doc(scoreDocs[i].doc);
+      assertTrue(Integer.parseInt(document.get("test.test"), radix) < 3);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java b/src/blur-query/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
new file mode 100644
index 0000000..c315933
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
@@ -0,0 +1,107 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.blur.index.IndexWriter;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class LongAnalyzerTest {
+
+  @Test
+  public void testLongAnalyzer() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("long", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 10);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStep() throws IOException {
+
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("long,4", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 10);
+  }
+
+  @Test
+  public void testLongAnalyzerDifferentStepAndRadix() throws IOException {
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
+    columnDefinitions.put("test", new ColumnDefinition("long,4,16", false, null));
+    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
+    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
+    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
+    runTestString(analyzer, 16);
+  }
+
+  private void runTestString(Analyzer analyzer, int radix) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
+    Directory dir = new RAMDirectory();
+    IndexWriter indexWriter = new IndexWriter(dir, conf);
+    for (int i = 0; i < 1000; i++) {
+      Document document = new Document();
+      String value = Long.toString(i, radix);
+      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
+    NumericRangeQuery<Long> query = NumericRangeQuery.newLongRange("test.test", 0L, 2L, true, true);
+    Query rewrite = searcher.rewrite(query);
+    TopDocs docs = searcher.search(rewrite, 100);
+    ScoreDoc[] scoreDocs = docs.scoreDocs;
+    assertEquals(3, docs.totalHits);
+    for (int i = 0; i < docs.totalHits; i++) {
+      Document document = searcher.doc(scoreDocs[i].doc);
+      assertTrue(Long.parseLong(document.get("test.test"), radix) < 3);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
new file mode 100644
index 0000000..707916c
--- /dev/null
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
@@ -0,0 +1,152 @@
+package org.apache.blur.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static junit.framework.Assert.assertTrue;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.blur.thrift.generated.ScoreType;
+import org.apache.blur.lucene.search.SuperQuery;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.RAMDirectory;
+import org.junit.Test;
+
+public class RandomSuperQueryTest {
+
+  private static final int MOD_COLS_USED_FOR_SKIPPING = 3;
+  private static final int MAX_NUM_OF_DOCS = 10000;// 10000
+  private static final int MIN_NUM_COL_FAM = 3;// 3
+  private static final int MAX_NUM_COL_FAM = 20;// 20
+  private static final int MAX_NUM_DOCS_PER_COL_FAM = 25;// 25
+  private static final int MAX_NUM_COLS = 21;// 21
+  private static final int MIN_NUM_COLS = 3;// 3
+  private static final int MAX_NUM_OF_WORDS = 1000;
+  private static final int MOD_USED_FOR_SAMPLING = 7;//
+  private static final String PRIME_DOC = "_p_";
+  private static final String PRIME_DOC_VALUE = "_true_";
+
+  private Random seedGen = new Random(1);
+
+  @Test
+  public void testRandomSuperQuery() throws CorruptIndexException, IOException, InterruptedException, ParseException {
+    long seed = seedGen.nextLong();
+
+    Random random = new Random(seed);
+    Collection<Query> sampler = new HashSet<Query>();
+    System.out.print("Creating index... ");
+    System.out.flush();
+    Directory directory = createIndex(random, sampler);
+    IndexReader reader = DirectoryReader.open(directory);
+    System.out.print("Running searches [" + sampler.size() + "]... ");
+    System.out.flush();
+    assertTrue(!sampler.isEmpty());
+    IndexSearcher searcher = new IndexSearcher(reader);
+    long s = System.currentTimeMillis();
+    for (Query query : sampler) {
+      TopDocs topDocs = searcher.search(query, 10);
+      assertTrue("seed [" + seed + "] {" + query + "} {" + s + "}", topDocs.totalHits > 0);
+    }
+    long e = System.currentTimeMillis();
+    System.out.println("Finished in [" + (e - s) + "] ms");
+  }
+
+  private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException {
+    Directory directory = new RAMDirectory();
+    String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
+    Map<String, String[]> columns = new HashMap<String, String[]>();
+    for (int i = 0; i < columnFamilies.length; i++) {
+      columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
+    }
+    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
+    int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
+    for (int i = 0; i < numberOfDocs; i++) {
+      writer.addDocuments(generatSuperDoc(random, columns, sampler));
+    }
+    writer.close();
+    return directory;
+  }
+
+  private String[] genWords(Random random, int min, int max, String prefix) {
+    int numberOfColFam = random.nextInt(max - min) + min;
+    String[] str = new String[numberOfColFam];
+    for (int i = 0; i < numberOfColFam; i++) {
+      str[i] = genWord(random, prefix);
+    }
+    return str;
+  }
+
+  private List<Document> generatSuperDoc(Random random, Map<String, String[]> columns, Collection<Query> sampler) {
+    List<Document> docs = new ArrayList<Document>();
+    BooleanQuery booleanQuery = new BooleanQuery();
+    for (String colFam : columns.keySet()) {
+      String[] cols = columns.get(colFam);
+      for (int i = 0; i < random.nextInt(MAX_NUM_DOCS_PER_COL_FAM); i++) {
+        Document doc = new Document();
+        for (String column : cols) {
+          if (random.nextInt() % MOD_COLS_USED_FOR_SKIPPING == 0) {
+            String word = genWord(random, "word");
+            doc.add(new StringField(colFam + "." + column, word, Store.YES));
+            if (random.nextInt() % MOD_USED_FOR_SAMPLING == 0) {
+              TermQuery termQuery = new TermQuery(new Term(colFam + "." + column, word));
+              SuperQuery query = new SuperQuery(termQuery, ScoreType.SUPER, new Term(PRIME_DOC, PRIME_DOC_VALUE));
+              booleanQuery.add(query, Occur.MUST);
+            }
+          }
+        }
+        docs.add(doc);
+      }
+    }
+    if (!booleanQuery.clauses().isEmpty()) {
+      sampler.add(booleanQuery);
+    }
+    Document document = docs.get(0);
+    document.add(new StringField(PRIME_DOC, PRIME_DOC_VALUE, Store.NO));
+    return docs;
+  }
+
+  private String genWord(Random random, String prefix) {
+    return prefix + random.nextInt(MAX_NUM_OF_WORDS);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
index 381ffe1..5a33b46 100644
--- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
@@ -1,28 +1,47 @@
 package org.apache.blur.lucene.search;
 
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.junit.Assert.*;
 
+import java.util.List;
+
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Version;
 import org.junit.Before;
 import org.junit.Test;
 
 public class SuperParserTest {
 
   private SuperParser parser;
+  private BlurAnalyzer analyzer;
 
   @Before
   public void setup() {
-    parser = new SuperParser(Version.LUCENE_40, "", new WhitespaceAnalyzer(Version.LUCENE_40), ScoreType.SUPER, new Term("_primedoc_"));
+    AnalyzerDefinition ad = new AnalyzerDefinition();
+    ad.setDefaultDefinition(new ColumnDefinition(StandardAnalyzer.class.getName(), true, null));
+    ColumnFamilyDefinition cfDef = new ColumnFamilyDefinition();
+    cfDef.putToColumnDefinitions("id_l", new ColumnDefinition("long", false, null));
+    cfDef.putToColumnDefinitions("id_d", new ColumnDefinition("double", false, null));
+    cfDef.putToColumnDefinitions("id_f", new ColumnDefinition("float", false, null));
+    cfDef.putToColumnDefinitions("id_i", new ColumnDefinition("integer", false, null));
+    ad.putToColumnFamilyDefinitions("a", cfDef);
+    analyzer = new BlurAnalyzer(ad);
+    parser = new SuperParser(LUCENE_VERSION, new BlurAnalyzer(new WhitespaceAnalyzer(LUCENE_VERSION)), true, null,  ScoreType.SUPER, new Term("_primedoc_"));
   }
 
   @Test
@@ -65,7 +84,6 @@ public class SuperParserTest {
     booleanQuery.add(new TermQuery(new Term("a", "a")), Occur.SHOULD);
     booleanQuery.add(new TermQuery(new Term("d", "e")), Occur.SHOULD);
     booleanQuery.add(new TermQuery(new Term("b", "b")), Occur.SHOULD);
-
     assertEquals(booleanQuery, query);
   }
 
@@ -120,7 +138,7 @@ public class SuperParserTest {
 
   @Test
   public void testParser6() throws ParseException {
-    SuperParser parser = new SuperParser(Version.LUCENE_40, "", new StandardAnalyzer(Version.LUCENE_40), ScoreType.SUPER, new Term("_primedoc_"));
+    SuperParser parser = new SuperParser(LUCENE_VERSION, analyzer, true, null,  ScoreType.SUPER, new Term("_primedoc_"));
     try {
       parser.parse("super : <a:a d:{e TO d} b:b super:<test:hello\\<>> super:<c:c d:d>");
       fail();
@@ -128,4 +146,191 @@ public class SuperParserTest {
       // should throw an error
     }
   }
+  
+  @Test
+  public void test7() throws ParseException {
+    Query q = parseSq("(a.b:cool) (+a.c:cool a.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc_m(tq("a.c", "cool")), bc(tq("a.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test8() throws ParseException {
+    Query q = parseSq("(a.b:cool) (a.c:cool a.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc(tq("a.c", "cool")), bc(tq("a.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test9() throws ParseException {
+    Query q = parseSq("a.b:cool (a.c:cool a.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc(tq("a.c", "cool")), bc(tq("a.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test10() throws ParseException {
+    Query q = parseSq("a.b:cool a.c:cool a.b:cool");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(tq("a.c", "cool")), bc(tq("a.b", "cool"))), q);
+  }
+
+  @Test
+  public void test11() throws ParseException {
+    Query q = parseSq("(a.b:cool) (+a.c:cool c.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc_m(tq("a.c", "cool")), bc(tq("c.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test12() throws ParseException {
+    Query q = parseSq("(a.b:cool) (a.c:cool c.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc(tq("a.c", "cool")), bc(tq("c.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test13() throws ParseException {
+    Query q = parseSq("a.b:cool (a.c:cool c.b:cool)");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(bq(bc(tq("a.c", "cool")), bc(tq("c.b", "cool"))))), q);
+  }
+
+  @Test
+  public void test14() throws ParseException {
+    Query q = parseSq("a.b:cool a.c:cool c.b:cool");
+    assertQuery(bq(bc(tq("a.b", "cool")), bc(tq("a.c", "cool")), bc(tq("c.b", "cool"))), q);
+  }
+
+  @Test
+  public void test15() throws ParseException {
+    Query q = parseSq("a.id_l:[0 TO 2]");
+    Query q1 = rq_i("a.id_l", 0L, 2L);
+    assertQuery(q1, q);
+  }
+  
+  @Test
+  public void test16() throws ParseException {
+    Query q = parseSq("a.id_d:[0 TO 2]");
+    assertQuery(rq_i("a.id_d", 0.0D, 2.0D), q);
+  }
+  
+  @Test
+  public void test17() throws ParseException {
+    Query q = parseSq("a.id_f:[0 TO 2]");
+    assertQuery(rq_i("a.id_f", 0.0F, 2.0F), q);
+  }
+  
+  @Test
+  public void test18() throws ParseException {
+    Query q = parseSq("a.id_i:[0 TO 2]");
+    Query q1 = rq_i("a.id_i", 0, 2);
+    assertQuery(q1, q);
+  }
+
+  public static BooleanClause bc_m(Query q) {
+    return new BooleanClause(q, Occur.MUST);
+  }
+
+  public static BooleanClause bc_n(Query q) {
+    return new BooleanClause(q, Occur.MUST_NOT);
+  }
+
+  public static BooleanClause bc(Query q) {
+    return new BooleanClause(q, Occur.SHOULD);
+  }
+
+  public static void assertQuery(Query expected, Query actual) {
+    System.out.println(expected);
+    System.out.println(actual);
+    assertEqualsQuery(expected, actual);
+  }
+
+  public static void assertEqualsQuery(Query expected, Query actual) {
+    assertEquals(expected.getClass(), actual.getClass());
+    if (expected instanceof BooleanQuery) {
+      assertEqualsBooleanQuery((BooleanQuery) expected, (BooleanQuery) actual);
+    } else if (expected instanceof SuperQuery) {
+      assertEqualsSuperQuery((SuperQuery) expected, (SuperQuery) actual);
+    } else if (expected instanceof TermQuery) {
+      assertEqualsTermQuery((TermQuery) expected, (TermQuery) actual);
+    } else if (expected instanceof NumericRangeQuery<?>) {
+      assertEqualsNumericRangeQuery((NumericRangeQuery<?>) expected, (NumericRangeQuery<?>) actual);
+    }
+    else {
+      fail("Type [" + expected.getClass() + "] not supported");
+    }
+  }
+
+  public static void assertEqualsTermQuery(TermQuery expected, TermQuery actual) {
+    Term term1 = expected.getTerm();
+    Term term2 = actual.getTerm();
+    assertEquals(term1, term2);
+  }
+
+  public static void assertEqualsNumericRangeQuery(NumericRangeQuery<?> expected, NumericRangeQuery<?> actual) {
+    assertEquals(expected, actual);
+  }
+  
+  public static void assertEqualsSuperQuery(SuperQuery expected, SuperQuery actual) {
+    assertEquals(expected.getQuery(), actual.getQuery());
+  }
+
+  public static void assertEqualsBooleanQuery(BooleanQuery expected, BooleanQuery actual) {
+    List<BooleanClause> clauses1 = expected.clauses();
+    List<BooleanClause> clauses2 = actual.clauses();
+    assertEqualsBooleanClause(clauses1, clauses2);
+  }
+
+  public static void assertEqualsBooleanClause(List<BooleanClause> clauses1, List<BooleanClause> clauses2) {
+    if (clauses1 == null && clauses2 == null) {
+      return;
+    }
+    if (clauses1 == null || clauses2 == null) {
+      fail();
+    }
+    if (clauses1.size() != clauses2.size()) {
+      fail();
+    }
+    int size = clauses1.size();
+    for (int i = 0; i < size; i++) {
+      assertEqualsBooleanClause(clauses1.get(i), clauses2.get(i));
+    }
+  }
+
+  public static void assertEqualsBooleanClause(BooleanClause booleanClause1, BooleanClause booleanClause2) {
+    assertEquals(booleanClause1.getOccur(), booleanClause2.getOccur());
+    assertEqualsQuery(booleanClause1.getQuery(), booleanClause2.getQuery());
+  }
+  
+  private Query rq_i(String field, float min, float max) {
+    return NumericRangeQuery.newFloatRange(field, min, max, true, true);
+  }
+  
+  private Query rq_i(String field, int min, int max) {
+    return NumericRangeQuery.newIntRange(field, min, max, true, true);
+  }
+  
+  private Query rq_i(String field, double min, double max) {
+    return NumericRangeQuery.newDoubleRange(field, min, max, true, true);
+  }
+
+  private Query rq_i(String field, long min, long max) {
+    return NumericRangeQuery.newLongRange(field, min, max, true, true);
+  }
+
+  private BooleanQuery bq(BooleanClause... bcs) {
+    BooleanQuery bq = new BooleanQuery();
+    for (BooleanClause bc : bcs) {
+      bq.add(bc);
+    }
+    return bq;
+  }
+
+  private SuperQuery sq(Query q) {
+    return new SuperQuery(q, ScoreType.SUPER, new Term("_primedoc_"));
+  }
+
+  private TermQuery tq(String field, String text) {
+    return new TermQuery(new Term(field, text));
+  }
+
+  private Query parseSq(String qstr) throws ParseException {
+    SuperParser superParser = new SuperParser(LUCENE_VERSION, analyzer, true, null,  ScoreType.SUPER, new Term("_primedoc_"));
+    return superParser.parse(qstr);
+  }
+  
 }


Mime
View raw message